public virtual void TestRMDownRestoreForJobStatusBeforeGetAMReport() { Configuration conf = new YarnConfiguration(); conf.SetInt(MRJobConfig.MrClientMaxRetries, 3); conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName); conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient); MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest >())).ThenReturn(GetJobReportResponse()); ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); try { Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenThrow (new UndeclaredThrowableException(new IOException("Connection refuced1"))).ThenThrow (new UndeclaredThrowableException(new IOException("Connection refuced2"))).ThenReturn (GetFinishedApplicationReport()); ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rmDelegate , oldJobId, historyServerProxy); JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); Org.Mockito.Mockito.Verify(rmDelegate, Org.Mockito.Mockito.Times(3)).GetApplicationReport (Matchers.Any <ApplicationId>()); NUnit.Framework.Assert.IsNotNull(jobStatus); } catch (YarnException e) { throw new IOException(e); } }
public virtual void TestRetriesOnAMConnectionFailures() { if (!isAMReachableFromClient) { return; } ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId ())).ThenReturn(GetRunningApplicationReport("am1", 78)); // throw exception in 1st, 2nd, 3rd and 4th call of getJobReport, and // succeed in the 5th call. MRClientProtocol amProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(); Org.Mockito.Mockito.When(amProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest >())).ThenThrow(new RuntimeException("11")).ThenThrow(new RuntimeException("22") ).ThenThrow(new RuntimeException("33")).ThenThrow(new RuntimeException("44")).ThenReturn (GetJobReportResponse()); Configuration conf = new YarnConfiguration(); conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName); conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient); ClientServiceDelegate clientServiceDelegate = new _ClientServiceDelegate_167(amProxy , conf, rm, oldJobId, null); JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); // assert maxClientRetry is not decremented. NUnit.Framework.Assert.AreEqual(conf.GetInt(MRJobConfig.MrClientMaxRetries, MRJobConfig .DefaultMrClientMaxRetries), clientServiceDelegate.GetMaxClientRetry()); Org.Mockito.Mockito.Verify(amProxy, Org.Mockito.Mockito.Times(5)).GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest>()); }
/// <exception cref="System.IO.IOException"/> private void TestRMDownForJobStatusBeforeGetAMReport(Configuration conf, int noOfRetries ) { conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName); conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient); MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); try { Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenThrow (new UndeclaredThrowableException(new IOException("Connection refuced1"))).ThenThrow (new UndeclaredThrowableException(new IOException("Connection refuced2"))).ThenThrow (new UndeclaredThrowableException(new IOException("Connection refuced3"))); ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rmDelegate , oldJobId, historyServerProxy); try { clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.Fail("It should throw exception after retries"); } catch (IOException e) { System.Console.Out.WriteLine("fail to get job status,and e=" + e.ToString()); } Org.Mockito.Mockito.Verify(rmDelegate, Org.Mockito.Mockito.Times(noOfRetries)).GetApplicationReport (Matchers.Any <ApplicationId>()); } catch (YarnException e) { throw new IOException(e); } }
public MRClientService(AppContext appContext) : base(typeof(Org.Apache.Hadoop.Mapreduce.V2.App.Client.MRClientService).FullName ) { this.appContext = appContext; this.protocolHandler = new MRClientService.MRClientProtocolHandler(this); }
//TODO: evict from the cache on some threshold public virtual ClientServiceDelegate GetClient(JobID jobId) { lock (this) { if (hsProxy == null) { try { hsProxy = InstantiateHistoryProxy(); } catch (IOException e) { Log.Warn("Could not connect to History server.", e); throw new YarnRuntimeException("Could not connect to History server.", e); } } ClientServiceDelegate client = cache[jobId]; if (client == null) { client = new ClientServiceDelegate(conf, rm, jobId, hsProxy); cache[jobId] = client; } return(client); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> private long RenewDelegationToken(UserGroupInformation loggedInUser, MRClientProtocol hsService, Token dToken) { long nextExpTime = loggedInUser.DoAs(new _PrivilegedExceptionAction_252(dToken, hsService )); return(nextExpTime); }
public virtual void TestAMAccessDisabled() { //test only applicable when AM not reachable if (isAMReachableFromClient) { return; } MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())). ThenReturn(GetJobReportResponseFromHistoryServer()); ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); try { Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenReturn (GetRunningApplicationReport("am1", 78)).ThenReturn(GetRunningApplicationReport( "am1", 78)).ThenReturn(GetRunningApplicationReport("am1", 78)).ThenReturn(GetFinishedApplicationReport ()); } catch (YarnException e) { throw new IOException(e); } ClientServiceDelegate clientServiceDelegate = Org.Mockito.Mockito.Spy(GetClientServiceDelegate (historyServerProxy, rmDelegate)); JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetJobName()); Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy (Matchers.Any <IPEndPoint>()); // Should not reach AM even for second and third times too. jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetJobName()); Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy (Matchers.Any <IPEndPoint>()); jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetJobName()); Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy (Matchers.Any <IPEndPoint>()); // The third time around, app is completed, so should go to JHS JobStatus jobStatus1 = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus1); NUnit.Framework.Assert.AreEqual("TestJobFilePath", jobStatus1.GetJobFile()); NUnit.Framework.Assert.AreEqual("http://TestTrackingUrl", jobStatus1.GetTrackingUrl ()); NUnit.Framework.Assert.AreEqual(1.0f, jobStatus1.GetMapProgress(), 0.0f); NUnit.Framework.Assert.AreEqual(1.0f, jobStatus1.GetReduceProgress(), 0.0f); Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy (Matchers.Any <IPEndPoint>()); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> private Token GetDelegationToken(UserGroupInformation loggedInUser, MRClientProtocol hsService, string renewerString) { // Get the delegation token directly as it is a little difficult to setup // the kerberos based rpc. Token token = loggedInUser.DoAs(new _PrivilegedExceptionAction_236(renewerString, hsService)); return(token); }
/// <exception cref="System.IO.IOException"/> protected internal virtual MRClientProtocol GetInitializedHSProxy() { lock (this) { if (this.hsProxy == null) { hsProxy = InstantiateHistoryProxy(); } return(this.hsProxy); } }
/// <exception cref="System.IO.IOException"/> internal virtual MRClientProtocol InstantiateAMProxy(IPEndPoint serviceAddr) { Log.Trace("Connecting to ApplicationMaster at: " + serviceAddr); YarnRPC rpc = YarnRPC.Create(conf); MRClientProtocol proxy = (MRClientProtocol)rpc.GetProxy(typeof(MRClientProtocol), serviceAddr, conf); usingAMProxy.Set(true); Log.Trace("Connected to ApplicationMaster at: " + serviceAddr); return(proxy); }
internal virtual Org.Apache.Hadoop.Security.Token.Token <object> GetDelegationTokenFromHS (MRClientProtocol hsProxy) { GetDelegationTokenRequest request = recordFactory.NewRecordInstance <GetDelegationTokenRequest >(); request.SetRenewer(Master.GetMasterPrincipal(conf)); Org.Apache.Hadoop.Yarn.Api.Records.Token mrDelegationToken; mrDelegationToken = hsProxy.GetDelegationToken(request).GetDelegationToken(); return(ConverterUtils.ConvertFromYarn(mrDelegationToken, hsProxy.GetConnectAddress ())); }
private ClientServiceDelegate GetClientServiceDelegate(MRClientProtocol historyServerProxy , ResourceMgrDelegate rm) { Configuration conf = new YarnConfiguration(); conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName); conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient); ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rm, oldJobId, historyServerProxy); return(clientServiceDelegate); }
private MRClientProtocol GetMRClientProtocol(Token token, IPEndPoint hsAddress, string user, Configuration conf) { UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(user); ugi.AddToken(ConverterUtils.ConvertFromYarn(token, hsAddress)); YarnRPC rpc = YarnRPC.Create(conf); MRClientProtocol hsWithDT = ugi.DoAs(new _PrivilegedAction_288(rpc, hsAddress, conf )); return(hsWithDT); }
public virtual void TestUnknownAppInRM() { MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())). ThenReturn(GetJobReportResponse()); ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy , GetRMDelegate()); JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); }
/// <exception cref="System.Exception"/> public virtual void TestHistoryServerToken() { //Set the master principal in the config conf.Set(YarnConfiguration.RmPrincipal, "foo@LOCAL"); string masterPrincipal = Master.GetMasterPrincipal(conf); MRClientProtocol hsProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(); Org.Mockito.Mockito.When(hsProxy.GetDelegationToken(Matchers.Any <GetDelegationTokenRequest >())).ThenAnswer(new _Answer_410(masterPrincipal)); // check that the renewer matches the cluster's RM principal // none of these fields matter for the sake of the test UserGroupInformation.CreateRemoteUser("someone").DoAs(new _PrivilegedExceptionAction_432 (this, hsProxy)); }
public ClientServiceDelegate(Configuration conf, ResourceMgrDelegate rm, JobID jobId , MRClientProtocol historyServerProxy) { // Caches for per-user NotRunningJobs this.conf = new Configuration(conf); // Cloning for modifying. // For faster redirects from AM to HS. this.conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectMaxRetriesKey, this .conf.GetInt(MRJobConfig.MrClientToAmIpcMaxRetries, MRJobConfig.DefaultMrClientToAmIpcMaxRetries )); this.conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectMaxRetriesOnSocketTimeoutsKey , this.conf.GetInt(MRJobConfig.MrClientToAmIpcMaxRetriesOnTimeouts, MRJobConfig. DefaultMrClientToAmIpcMaxRetriesOnTimeouts)); this.rm = rm; this.jobId = jobId; this.historyServerProxy = historyServerProxy; this.appId = TypeConverter.ToYarn(jobId).GetAppId(); notRunningJobs = new Dictionary <JobState, Dictionary <string, NotRunningJob> >(); }
public virtual void TestCountersFromHistoryServer() { MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); Org.Mockito.Mockito.When(historyServerProxy.GetCounters(GetCountersRequest())).ThenReturn (GetCountersResponseFromHistoryServer()); ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId ())).ThenReturn(null); ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy , rm); Counters counters = TypeConverter.ToYarn(clientServiceDelegate.GetJobCounters(oldJobId )); NUnit.Framework.Assert.IsNotNull(counters); NUnit.Framework.Assert.AreEqual(1001, counters.GetCounterGroup("dummyCounters").GetCounter ("dummyCounter").GetValue()); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public override void Cancel <_T0>(Org.Apache.Hadoop.Security.Token.Token <_T0> token , Configuration conf) { Org.Apache.Hadoop.Yarn.Api.Records.Token dToken = Org.Apache.Hadoop.Yarn.Api.Records.Token .NewInstance(token.GetIdentifier(), token.GetKind().ToString(), token.GetPassword (), token.GetService().ToString()); MRClientProtocol histProxy = InstantiateHistoryProxy(conf, SecurityUtil.GetTokenServiceAddr (token)); try { CancelDelegationTokenRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <CancelDelegationTokenRequest>(); request.SetDelegationToken(dToken); histProxy.CancelDelegationToken(request); } finally { StopHistoryProxy(histProxy); } }
public virtual void TestRetriesOnConnectionFailure() { MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())). ThenThrow(new RuntimeException("1")).ThenThrow(new RuntimeException("2")).ThenReturn (GetJobReportResponse()); ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId ())).ThenReturn(null); ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy , rm); JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); Org.Mockito.Mockito.Verify(historyServerProxy, Org.Mockito.Mockito.Times(3)).GetJobReport (Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest >()); }
public virtual void TestJobReportFromHistoryServer() { MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())). ThenReturn(GetJobReportResponseFromHistoryServer()); ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId ())).ThenReturn(null); ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy , rm); JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); NUnit.Framework.Assert.AreEqual("TestJobFilePath", jobStatus.GetJobFile()); NUnit.Framework.Assert.AreEqual("http://TestTrackingUrl", jobStatus.GetTrackingUrl ()); NUnit.Framework.Assert.AreEqual(1.0f, jobStatus.GetMapProgress(), 0.0f); NUnit.Framework.Assert.AreEqual(1.0f, jobStatus.GetReduceProgress(), 0.0f); }
public virtual void TestNoRetryOnAMAuthorizationException() { if (!isAMReachableFromClient) { return; } ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId ())).ThenReturn(GetRunningApplicationReport("am1", 78)); // throw authorization exception on first invocation MRClientProtocol amProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(); Org.Mockito.Mockito.When(amProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest >())).ThenThrow(new AuthorizationException("Denied")); Configuration conf = new YarnConfiguration(); conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName); conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient); ClientServiceDelegate clientServiceDelegate = new _ClientServiceDelegate_205(amProxy , conf, rm, oldJobId, null); try { clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.Fail("Exception should be thrown upon AuthorizationException" ); } catch (IOException e) { NUnit.Framework.Assert.AreEqual(typeof(AuthorizationException).FullName + ": Denied" , e.Message); } // assert maxClientRetry is not decremented. NUnit.Framework.Assert.AreEqual(conf.GetInt(MRJobConfig.MrClientMaxRetries, MRJobConfig .DefaultMrClientMaxRetries), clientServiceDelegate.GetMaxClientRetry()); Org.Mockito.Mockito.Verify(amProxy, Org.Mockito.Mockito.Times(1)).GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest>()); }
private void TestPbClientFactory() { IPEndPoint addr = new IPEndPoint(0); System.Console.Error.WriteLine(addr.GetHostName() + addr.Port); Configuration conf = new Configuration(); MRClientProtocol instance = new TestRPCFactories.MRClientProtocolTestImpl(this); Server server = null; try { server = RpcServerFactoryPBImpl.Get().GetServer(typeof(MRClientProtocol), instance , addr, conf, null, 1); server.Start(); System.Console.Error.WriteLine(server.GetListenerAddress()); System.Console.Error.WriteLine(NetUtils.GetConnectAddress(server)); MRClientProtocol client = null; try { client = (MRClientProtocol)RpcClientFactoryPBImpl.Get().GetClient(typeof(MRClientProtocol ), 1, NetUtils.GetConnectAddress(server), conf); } catch (YarnRuntimeException e) { Sharpen.Runtime.PrintStackTrace(e); NUnit.Framework.Assert.Fail("Failed to crete client"); } } catch (YarnRuntimeException e) { Sharpen.Runtime.PrintStackTrace(e); NUnit.Framework.Assert.Fail("Failed to crete server"); } finally { server.Stop(); } }
internal virtual void AddHistoryToken(Credentials ts) { /* check if we have a hsproxy, if not, no need */ MRClientProtocol hsProxy = clientCache.GetInitializedHSProxy(); if (UserGroupInformation.IsSecurityEnabled() && (hsProxy != null)) { /* * note that get delegation token was called. Again this is hack for oozie * to make sure we add history server delegation tokens to the credentials */ RMDelegationTokenSelector tokenSelector = new RMDelegationTokenSelector(); Text service = resMgrDelegate.GetRMDelegationTokenService(); if (tokenSelector.SelectToken(service, ts.GetAllTokens()) != null) { Text hsService = SecurityUtil.BuildTokenService(hsProxy.GetConnectAddress()); if (ts.GetToken(hsService) == null) { ts.AddToken(hsService, GetDelegationTokenFromHS(hsProxy)); } } } }
public virtual void TestRemoteExceptionFromHistoryServer() { MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())). ThenThrow(new IOException("Job ID doesnot Exist")); ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId ())).ThenReturn(null); ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy , rm); try { clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.Fail("Invoke should throw exception after retries."); } catch (IOException e) { NUnit.Framework.Assert.IsTrue(e.Message.Contains("Job ID doesnot Exist")); } }
/// <exception cref="System.IO.IOException"/> private object Invoke(string method, Type argClass, object args) { lock (this) { MethodInfo methodOb = null; try { methodOb = typeof(MRClientProtocol).GetMethod(method, argClass); } catch (SecurityException e) { throw new YarnRuntimeException(e); } catch (MissingMethodException e) { throw new YarnRuntimeException("Method name mismatch", e); } maxClientRetry = this.conf.GetInt(MRJobConfig.MrClientMaxRetries, MRJobConfig.DefaultMrClientMaxRetries ); IOException lastException = null; while (maxClientRetry > 0) { MRClientProtocol MRClientProxy = null; try { MRClientProxy = GetProxy(); return(methodOb.Invoke(MRClientProxy, args)); } catch (TargetInvocationException e) { // Will not throw out YarnException anymore Log.Debug("Failed to contact AM/History for job " + jobId + " retrying..", e.InnerException ); // Force reconnection by setting the proxy to null. realProxy = null; // HS/AMS shut down if (e.InnerException is AuthorizationException) { throw new IOException(e.InnerException); } // if it's AM shut down, do not decrement maxClientRetry as we wait for // AM to be restarted. if (!usingAMProxy.Get()) { maxClientRetry--; } usingAMProxy.Set(false); lastException = new IOException(e.InnerException); try { Sharpen.Thread.Sleep(100); } catch (Exception ie) { Log.Warn("ClientServiceDelegate invoke call interrupted", ie); throw new YarnRuntimeException(ie); } } catch (Exception e) { Log.Debug("Failed to contact AM/History for job " + jobId + " Will retry..", e); // Force reconnection by setting the proxy to null. realProxy = null; // RM shutdown maxClientRetry--; lastException = new IOException(e.Message); try { Sharpen.Thread.Sleep(100); } catch (Exception ie) { Log.Warn("ClientServiceDelegate invoke call interrupted", ie); throw new YarnRuntimeException(ie); } } } throw lastException; } }
public virtual void Test() { TestMRClientService.MRAppWithClientService app = new TestMRClientService.MRAppWithClientService (this, 1, 0, false); Configuration conf = new Configuration(); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task task = it.Next(); app.WaitForState(task, TaskState.Running); TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next(); app.WaitForState(attempt, TaskAttemptState.Running); // send the diagnostic string diagnostic1 = "Diagnostic1"; string diagnostic2 = "Diagnostic2"; app.GetContext().GetEventHandler().Handle(new TaskAttemptDiagnosticsUpdateEvent(attempt .GetID(), diagnostic1)); // send the status update TaskAttemptStatusUpdateEvent.TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus (); taskAttemptStatus.id = attempt.GetID(); taskAttemptStatus.progress = 0.5f; taskAttemptStatus.stateString = "RUNNING"; taskAttemptStatus.taskState = TaskAttemptState.Running; taskAttemptStatus.phase = Phase.Map; // send the status update app.GetContext().GetEventHandler().Handle(new TaskAttemptStatusUpdateEvent(attempt .GetID(), taskAttemptStatus)); //verify that all object are fully populated by invoking RPCs. YarnRPC rpc = YarnRPC.Create(conf); MRClientProtocol proxy = (MRClientProtocol)rpc.GetProxy(typeof(MRClientProtocol), app.clientService.GetBindAddress(), conf); GetCountersRequest gcRequest = recordFactory.NewRecordInstance <GetCountersRequest >(); gcRequest.SetJobId(job.GetID()); NUnit.Framework.Assert.IsNotNull("Counters is null", proxy.GetCounters(gcRequest) .GetCounters()); GetJobReportRequest gjrRequest = recordFactory.NewRecordInstance <GetJobReportRequest >(); gjrRequest.SetJobId(job.GetID()); JobReport jr = proxy.GetJobReport(gjrRequest).GetJobReport(); VerifyJobReport(jr); GetTaskAttemptCompletionEventsRequest gtaceRequest = recordFactory.NewRecordInstance <GetTaskAttemptCompletionEventsRequest>(); gtaceRequest.SetJobId(job.GetID()); gtaceRequest.SetFromEventId(0); gtaceRequest.SetMaxEvents(10); NUnit.Framework.Assert.IsNotNull("TaskCompletionEvents is null", proxy.GetTaskAttemptCompletionEvents (gtaceRequest).GetCompletionEventList()); GetDiagnosticsRequest gdRequest = recordFactory.NewRecordInstance <GetDiagnosticsRequest >(); gdRequest.SetTaskAttemptId(attempt.GetID()); NUnit.Framework.Assert.IsNotNull("Diagnostics is null", proxy.GetDiagnostics(gdRequest ).GetDiagnosticsList()); GetTaskAttemptReportRequest gtarRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest >(); gtarRequest.SetTaskAttemptId(attempt.GetID()); TaskAttemptReport tar = proxy.GetTaskAttemptReport(gtarRequest).GetTaskAttemptReport (); VerifyTaskAttemptReport(tar); GetTaskReportRequest gtrRequest = recordFactory.NewRecordInstance <GetTaskReportRequest >(); gtrRequest.SetTaskId(task.GetID()); NUnit.Framework.Assert.IsNotNull("TaskReport is null", proxy.GetTaskReport(gtrRequest ).GetTaskReport()); GetTaskReportsRequest gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest >(); gtreportsRequest.SetJobId(job.GetID()); gtreportsRequest.SetTaskType(TaskType.Map); NUnit.Framework.Assert.IsNotNull("TaskReports for map is null", proxy.GetTaskReports (gtreportsRequest).GetTaskReportList()); gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>(); gtreportsRequest.SetJobId(job.GetID()); gtreportsRequest.SetTaskType(TaskType.Reduce); NUnit.Framework.Assert.IsNotNull("TaskReports for reduce is null", proxy.GetTaskReports (gtreportsRequest).GetTaskReportList()); IList <string> diag = proxy.GetDiagnostics(gdRequest).GetDiagnosticsList(); NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, diag.Count); NUnit.Framework.Assert.AreEqual("Diag 1 not correct", diagnostic1, diag[0].ToString ()); TaskReport taskReport = proxy.GetTaskReport(gtrRequest).GetTaskReport(); NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, taskReport.GetDiagnosticsCount ()); //send the done signal to the task app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task.GetAttempts() .Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone)); app.WaitForState(job, JobState.Succeeded); // For invalid jobid, throw IOException gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>(); gtreportsRequest.SetJobId(TypeConverter.ToYarn(JobID.ForName("job_1415730144495_0001" ))); gtreportsRequest.SetTaskType(TaskType.Reduce); try { proxy.GetTaskReports(gtreportsRequest); NUnit.Framework.Assert.Fail("IOException not thrown for invalid job id"); } catch (IOException) { } }
public virtual void TestViewAclOnlyCannotModify() { TestMRClientService.MRAppWithClientService app = new TestMRClientService.MRAppWithClientService (this, 1, 0, false); Configuration conf = new Configuration(); conf.SetBoolean(MRConfig.MrAclsEnabled, true); conf.Set(MRJobConfig.JobAclViewJob, "viewonlyuser"); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task task = it.Next(); app.WaitForState(task, TaskState.Running); TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next(); app.WaitForState(attempt, TaskAttemptState.Running); UserGroupInformation viewOnlyUser = UserGroupInformation.CreateUserForTesting("viewonlyuser" , new string[] { }); NUnit.Framework.Assert.IsTrue("viewonlyuser cannot view job", job.CheckAccess(viewOnlyUser , JobACL.ViewJob)); NUnit.Framework.Assert.IsFalse("viewonlyuser can modify job", job.CheckAccess(viewOnlyUser , JobACL.ModifyJob)); MRClientProtocol client = viewOnlyUser.DoAs(new _PrivilegedExceptionAction_223(conf , app)); KillJobRequest killJobRequest = recordFactory.NewRecordInstance <KillJobRequest>(); killJobRequest.SetJobId(app.GetJobId()); try { client.KillJob(killJobRequest); NUnit.Framework.Assert.Fail("viewonlyuser killed job"); } catch (AccessControlException) { } // pass KillTaskRequest killTaskRequest = recordFactory.NewRecordInstance <KillTaskRequest >(); killTaskRequest.SetTaskId(task.GetID()); try { client.KillTask(killTaskRequest); NUnit.Framework.Assert.Fail("viewonlyuser killed task"); } catch (AccessControlException) { } // pass KillTaskAttemptRequest killTaskAttemptRequest = recordFactory.NewRecordInstance <KillTaskAttemptRequest >(); killTaskAttemptRequest.SetTaskAttemptId(attempt.GetID()); try { client.KillTaskAttempt(killTaskAttemptRequest); NUnit.Framework.Assert.Fail("viewonlyuser killed task attempt"); } catch (AccessControlException) { } // pass FailTaskAttemptRequest failTaskAttemptRequest = recordFactory.NewRecordInstance <FailTaskAttemptRequest >(); failTaskAttemptRequest.SetTaskAttemptId(attempt.GetID()); try { client.FailTaskAttempt(failTaskAttemptRequest); NUnit.Framework.Assert.Fail("viewonlyuser killed task attempt"); } catch (AccessControlException) { } }
public _PrivilegedExceptionAction_432(TestYARNRunner _enclosing, MRClientProtocol hsProxy) { this._enclosing = _enclosing; this.hsProxy = hsProxy; }
/// <exception cref="System.Exception"/> public virtual void TestGetHSDelegationToken() { try { Configuration conf = new Configuration(); // Setup mock service IPEndPoint mockRmAddress = new IPEndPoint("localhost", 4444); Text rmTokenSevice = SecurityUtil.BuildTokenService(mockRmAddress); IPEndPoint mockHsAddress = new IPEndPoint("localhost", 9200); Text hsTokenSevice = SecurityUtil.BuildTokenService(mockHsAddress); // Setup mock rm token RMDelegationTokenIdentifier tokenIdentifier = new RMDelegationTokenIdentifier(new Text("owner"), new Text("renewer"), new Text("real")); Org.Apache.Hadoop.Security.Token.Token <RMDelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token <RMDelegationTokenIdentifier>(new byte[0], new byte[0], tokenIdentifier.GetKind( ), rmTokenSevice); token.SetKind(RMDelegationTokenIdentifier.KindName); // Setup mock history token Org.Apache.Hadoop.Yarn.Api.Records.Token historyToken = Org.Apache.Hadoop.Yarn.Api.Records.Token .NewInstance(new byte[0], MRDelegationTokenIdentifier.KindName.ToString(), new byte [0], hsTokenSevice.ToString()); GetDelegationTokenResponse getDtResponse = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetDelegationTokenResponse>(); getDtResponse.SetDelegationToken(historyToken); // mock services MRClientProtocol mockHsProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(); Org.Mockito.Mockito.DoReturn(mockHsAddress).When(mockHsProxy).GetConnectAddress(); Org.Mockito.Mockito.DoReturn(getDtResponse).When(mockHsProxy).GetDelegationToken( Matchers.Any <GetDelegationTokenRequest>()); ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); Org.Mockito.Mockito.DoReturn(rmTokenSevice).When(rmDelegate).GetRMDelegationTokenService (); ClientCache clientCache = Org.Mockito.Mockito.Mock <ClientCache>(); Org.Mockito.Mockito.DoReturn(mockHsProxy).When(clientCache).GetInitializedHSProxy (); Credentials creds = new Credentials(); YARNRunner yarnRunner = new YARNRunner(conf, rmDelegate, clientCache); // No HS token if no RM token yarnRunner.AddHistoryToken(creds); Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(0)).GetDelegationToken (Matchers.Any <GetDelegationTokenRequest>()); // No HS token if RM token, but secirity disabled. creds.AddToken(new Text("rmdt"), token); yarnRunner.AddHistoryToken(creds); Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(0)).GetDelegationToken (Matchers.Any <GetDelegationTokenRequest>()); conf.Set(CommonConfigurationKeys.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); creds = new Credentials(); // No HS token if no RM token, security enabled yarnRunner.AddHistoryToken(creds); Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(0)).GetDelegationToken (Matchers.Any <GetDelegationTokenRequest>()); // HS token if RM token present, security enabled creds.AddToken(new Text("rmdt"), token); yarnRunner.AddHistoryToken(creds); Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(1)).GetDelegationToken (Matchers.Any <GetDelegationTokenRequest>()); // No additional call to get HS token if RM and HS token present yarnRunner.AddHistoryToken(creds); Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(1)).GetDelegationToken (Matchers.Any <GetDelegationTokenRequest>()); } finally { // Back to defaults. UserGroupInformation.SetConfiguration(new Configuration()); } }
//Test reports of JobHistoryServer. History server should get log files from MRApp and read them /// <exception cref="System.Exception"/> public virtual void TestReports() { Configuration config = new Configuration(); config.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey , typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping)); RackResolver.Init(config); MRApp app = new TestJobHistoryEvents.MRAppWithHistory(1, 1, true, this.GetType(). FullName, true); app.Submit(config); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); app.WaitForState(job, JobState.Succeeded); historyServer = new JobHistoryServer(); historyServer.Init(config); historyServer.Start(); // search JobHistory service JobHistory jobHistory = null; foreach (Org.Apache.Hadoop.Service.Service service in historyServer.GetServices()) { if (service is JobHistory) { jobHistory = (JobHistory)service; } } IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobs = jobHistory. GetAllJobs(); NUnit.Framework.Assert.AreEqual(1, jobs.Count); NUnit.Framework.Assert.AreEqual("job_0_0000", jobs.Keys.GetEnumerator().Next().ToString ()); Task task = job.GetTasks().Values.GetEnumerator().Next(); TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next(); HistoryClientService historyService = historyServer.GetClientService(); MRClientProtocol protocol = historyService.GetClientHandler(); GetTaskAttemptReportRequest gtarRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest >(); // test getTaskAttemptReport TaskAttemptId taId = attempt.GetID(); taId.SetTaskId(task.GetID()); taId.GetTaskId().SetJobId(job.GetID()); gtarRequest.SetTaskAttemptId(taId); GetTaskAttemptReportResponse response = protocol.GetTaskAttemptReport(gtarRequest ); NUnit.Framework.Assert.AreEqual("container_0_0000_01_000000", response.GetTaskAttemptReport ().GetContainerId().ToString()); NUnit.Framework.Assert.IsTrue(response.GetTaskAttemptReport().GetDiagnosticInfo() .IsEmpty()); // counters NUnit.Framework.Assert.IsNotNull(response.GetTaskAttemptReport().GetCounters().GetCounter (TaskCounter.PhysicalMemoryBytes)); NUnit.Framework.Assert.AreEqual(taId.ToString(), response.GetTaskAttemptReport(). GetTaskAttemptId().ToString()); // test getTaskReport GetTaskReportRequest request = recordFactory.NewRecordInstance <GetTaskReportRequest >(); TaskId taskId = task.GetID(); taskId.SetJobId(job.GetID()); request.SetTaskId(taskId); GetTaskReportResponse reportResponse = protocol.GetTaskReport(request); NUnit.Framework.Assert.AreEqual(string.Empty, reportResponse.GetTaskReport().GetDiagnosticsList ().GetEnumerator().Next()); // progress NUnit.Framework.Assert.AreEqual(1.0f, reportResponse.GetTaskReport().GetProgress( ), 0.01); // report has corrected taskId NUnit.Framework.Assert.AreEqual(taskId.ToString(), reportResponse.GetTaskReport() .GetTaskId().ToString()); // Task state should be SUCCEEDED NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, reportResponse.GetTaskReport ().GetTaskState()); // For invalid jobid, throw IOException GetTaskReportsRequest gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest >(); gtreportsRequest.SetJobId(TypeConverter.ToYarn(JobID.ForName("job_1415730144495_0001" ))); gtreportsRequest.SetTaskType(TaskType.Reduce); try { protocol.GetTaskReports(gtreportsRequest); NUnit.Framework.Assert.Fail("IOException not thrown for invalid job id"); } catch (IOException) { } // Expected // test getTaskAttemptCompletionEvents GetTaskAttemptCompletionEventsRequest taskAttemptRequest = recordFactory.NewRecordInstance <GetTaskAttemptCompletionEventsRequest>(); taskAttemptRequest.SetJobId(job.GetID()); GetTaskAttemptCompletionEventsResponse taskAttemptCompletionEventsResponse = protocol .GetTaskAttemptCompletionEvents(taskAttemptRequest); NUnit.Framework.Assert.AreEqual(0, taskAttemptCompletionEventsResponse.GetCompletionEventCount ()); // test getDiagnostics GetDiagnosticsRequest diagnosticRequest = recordFactory.NewRecordInstance <GetDiagnosticsRequest >(); diagnosticRequest.SetTaskAttemptId(taId); GetDiagnosticsResponse diagnosticResponse = protocol.GetDiagnostics(diagnosticRequest ); // it is strange : why one empty string ? NUnit.Framework.Assert.AreEqual(1, diagnosticResponse.GetDiagnosticsCount()); NUnit.Framework.Assert.AreEqual(string.Empty, diagnosticResponse.GetDiagnostics(0 )); }