public virtual void TestRMDownRestoreForJobStatusBeforeGetAMReport() { Configuration conf = new YarnConfiguration(); conf.SetInt(MRJobConfig.MrClientMaxRetries, 3); conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName); conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient); MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest >())).ThenReturn(GetJobReportResponse()); ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); try { Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenThrow (new UndeclaredThrowableException(new IOException("Connection refuced1"))).ThenThrow (new UndeclaredThrowableException(new IOException("Connection refuced2"))).ThenReturn (GetFinishedApplicationReport()); ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rmDelegate , oldJobId, historyServerProxy); JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); Org.Mockito.Mockito.Verify(rmDelegate, Org.Mockito.Mockito.Times(3)).GetApplicationReport (Matchers.Any <ApplicationId>()); NUnit.Framework.Assert.IsNotNull(jobStatus); } catch (YarnException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> private void TestRMDownForJobStatusBeforeGetAMReport(Configuration conf, int noOfRetries ) { conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName); conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient); MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); try { Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenThrow (new UndeclaredThrowableException(new IOException("Connection refuced1"))).ThenThrow (new UndeclaredThrowableException(new IOException("Connection refuced2"))).ThenThrow (new UndeclaredThrowableException(new IOException("Connection refuced3"))); ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rmDelegate , oldJobId, historyServerProxy); try { clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.Fail("It should throw exception after retries"); } catch (IOException e) { System.Console.Out.WriteLine("fail to get job status,and e=" + e.ToString()); } Org.Mockito.Mockito.Verify(rmDelegate, Org.Mockito.Mockito.Times(noOfRetries)).GetApplicationReport (Matchers.Any <ApplicationId>()); } catch (YarnException e) { throw new IOException(e); } }
public virtual void TestRetriesOnAMConnectionFailures() { if (!isAMReachableFromClient) { return; } ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId ())).ThenReturn(GetRunningApplicationReport("am1", 78)); // throw exception in 1st, 2nd, 3rd and 4th call of getJobReport, and // succeed in the 5th call. MRClientProtocol amProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(); Org.Mockito.Mockito.When(amProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest >())).ThenThrow(new RuntimeException("11")).ThenThrow(new RuntimeException("22") ).ThenThrow(new RuntimeException("33")).ThenThrow(new RuntimeException("44")).ThenReturn (GetJobReportResponse()); Configuration conf = new YarnConfiguration(); conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName); conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient); ClientServiceDelegate clientServiceDelegate = new _ClientServiceDelegate_167(amProxy , conf, rm, oldJobId, null); JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); // assert maxClientRetry is not decremented. NUnit.Framework.Assert.AreEqual(conf.GetInt(MRJobConfig.MrClientMaxRetries, MRJobConfig .DefaultMrClientMaxRetries), clientServiceDelegate.GetMaxClientRetry()); Org.Mockito.Mockito.Verify(amProxy, Org.Mockito.Mockito.Times(5)).GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest>()); }
public virtual void TestAMAccessDisabled() { //test only applicable when AM not reachable if (isAMReachableFromClient) { return; } MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())). ThenReturn(GetJobReportResponseFromHistoryServer()); ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); try { Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenReturn (GetRunningApplicationReport("am1", 78)).ThenReturn(GetRunningApplicationReport( "am1", 78)).ThenReturn(GetRunningApplicationReport("am1", 78)).ThenReturn(GetFinishedApplicationReport ()); } catch (YarnException e) { throw new IOException(e); } ClientServiceDelegate clientServiceDelegate = Org.Mockito.Mockito.Spy(GetClientServiceDelegate (historyServerProxy, rmDelegate)); JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetJobName()); Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy (Matchers.Any <IPEndPoint>()); // Should not reach AM even for second and third times too. jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetJobName()); Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy (Matchers.Any <IPEndPoint>()); jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetJobName()); Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy (Matchers.Any <IPEndPoint>()); // The third time around, app is completed, so should go to JHS JobStatus jobStatus1 = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus1); NUnit.Framework.Assert.AreEqual("TestJobFilePath", jobStatus1.GetJobFile()); NUnit.Framework.Assert.AreEqual("http://TestTrackingUrl", jobStatus1.GetTrackingUrl ()); NUnit.Framework.Assert.AreEqual(1.0f, jobStatus1.GetMapProgress(), 0.0f); NUnit.Framework.Assert.AreEqual(1.0f, jobStatus1.GetReduceProgress(), 0.0f); Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy (Matchers.Any <IPEndPoint>()); }
private ClientServiceDelegate GetClientServiceDelegate(MRClientProtocol historyServerProxy , ResourceMgrDelegate rm) { Configuration conf = new YarnConfiguration(); conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName); conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient); ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rm, oldJobId, historyServerProxy); return(clientServiceDelegate); }
/// <summary> /// Similar to /// <see cref="YARNRunner(Org.Apache.Hadoop.Conf.Configuration, ResourceMgrDelegate)" /// /> /// but allowing injecting /// <see cref="ClientCache"/> /// . Enable mocking and testing. /// </summary> /// <param name="conf">the configuration object</param> /// <param name="resMgrDelegate">the resource manager delegate</param> /// <param name="clientCache">the client cache object.</param> public YARNRunner(Configuration conf, ResourceMgrDelegate resMgrDelegate, ClientCache clientCache) { this.conf = conf; try { this.resMgrDelegate = resMgrDelegate; this.clientCache = clientCache; this.defaultFileContext = FileContext.GetFileContext(this.conf); } catch (UnsupportedFileSystemException ufe) { throw new RuntimeException("Error in instantiating YarnClient", ufe); } }
/// <exception cref="System.IO.IOException"/> private ResourceMgrDelegate GetRMDelegate() { ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); try { ApplicationId appId = jobId.GetAppId(); Org.Mockito.Mockito.When(rm.GetApplicationReport(appId)).ThenThrow(new ApplicationNotFoundException (appId + " not found")); } catch (YarnException e) { throw new IOException(e); } return(rm); }
public ClientServiceDelegate(Configuration conf, ResourceMgrDelegate rm, JobID jobId , MRClientProtocol historyServerProxy) { // Caches for per-user NotRunningJobs this.conf = new Configuration(conf); // Cloning for modifying. // For faster redirects from AM to HS. this.conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectMaxRetriesKey, this .conf.GetInt(MRJobConfig.MrClientToAmIpcMaxRetries, MRJobConfig.DefaultMrClientToAmIpcMaxRetries )); this.conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectMaxRetriesOnSocketTimeoutsKey , this.conf.GetInt(MRJobConfig.MrClientToAmIpcMaxRetriesOnTimeouts, MRJobConfig. DefaultMrClientToAmIpcMaxRetriesOnTimeouts)); this.rm = rm; this.jobId = jobId; this.historyServerProxy = historyServerProxy; this.appId = TypeConverter.ToYarn(jobId).GetAppId(); notRunningJobs = new Dictionary <JobState, Dictionary <string, NotRunningJob> >(); }
public virtual void TestCountersFromHistoryServer() { MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); Org.Mockito.Mockito.When(historyServerProxy.GetCounters(GetCountersRequest())).ThenReturn (GetCountersResponseFromHistoryServer()); ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId ())).ThenReturn(null); ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy , rm); Counters counters = TypeConverter.ToYarn(clientServiceDelegate.GetJobCounters(oldJobId )); NUnit.Framework.Assert.IsNotNull(counters); NUnit.Framework.Assert.AreEqual(1001, counters.GetCounterGroup("dummyCounters").GetCounter ("dummyCounter").GetValue()); }
public virtual void TestRetriesOnConnectionFailure() { MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())). ThenThrow(new RuntimeException("1")).ThenThrow(new RuntimeException("2")).ThenReturn (GetJobReportResponse()); ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId ())).ThenReturn(null); ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy , rm); JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); Org.Mockito.Mockito.Verify(historyServerProxy, Org.Mockito.Mockito.Times(3)).GetJobReport (Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest >()); }
public virtual void TestHistoryServerNotConfigured() { //RM doesn't have app report and job History Server is not configured ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(null, GetRMDelegate ()); JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetUsername()); NUnit.Framework.Assert.AreEqual(JobStatus.State.Prep, jobStatus.GetState()); //RM has app report and job History Server is not configured ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); ApplicationReport applicationReport = GetFinishedApplicationReport(); Org.Mockito.Mockito.When(rm.GetApplicationReport(jobId.GetAppId())).ThenReturn(applicationReport ); clientServiceDelegate = GetClientServiceDelegate(null, rm); jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.AreEqual(applicationReport.GetUser(), jobStatus.GetUsername ()); NUnit.Framework.Assert.AreEqual(JobStatus.State.Succeeded, jobStatus.GetState()); }
protected override void SetUp() { resourceMgrDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); conf = new YarnConfiguration(); conf.Set(YarnConfiguration.RmPrincipal, "mapred/host@REALM"); clientCache = new ClientCache(conf, resourceMgrDelegate); clientCache = Org.Mockito.Mockito.Spy(clientCache); yarnRunner = new YARNRunner(conf, resourceMgrDelegate, clientCache); yarnRunner = Org.Mockito.Mockito.Spy(yarnRunner); submissionContext = Org.Mockito.Mockito.Mock <ApplicationSubmissionContext>(); Org.Mockito.Mockito.DoAnswer(new _Answer_146(this)).When(yarnRunner).CreateApplicationSubmissionContext (Matchers.Any <Configuration>(), Matchers.Any <string>(), Matchers.Any <Credentials >()); appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 1); jobId = TypeConverter.FromYarn(appId); if (testWorkDir.Exists()) { FileContext.GetLocalFSFileContext().Delete(new Path(testWorkDir.ToString()), true ); } testWorkDir.Mkdirs(); }
public virtual void TestNoRetryOnAMAuthorizationException() { if (!isAMReachableFromClient) { return; } ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId ())).ThenReturn(GetRunningApplicationReport("am1", 78)); // throw authorization exception on first invocation MRClientProtocol amProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(); Org.Mockito.Mockito.When(amProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest >())).ThenThrow(new AuthorizationException("Denied")); Configuration conf = new YarnConfiguration(); conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName); conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient); ClientServiceDelegate clientServiceDelegate = new _ClientServiceDelegate_205(amProxy , conf, rm, oldJobId, null); try { clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.Fail("Exception should be thrown upon AuthorizationException" ); } catch (IOException e) { NUnit.Framework.Assert.AreEqual(typeof(AuthorizationException).FullName + ": Denied" , e.Message); } // assert maxClientRetry is not decremented. NUnit.Framework.Assert.AreEqual(conf.GetInt(MRJobConfig.MrClientMaxRetries, MRJobConfig .DefaultMrClientMaxRetries), clientServiceDelegate.GetMaxClientRetry()); Org.Mockito.Mockito.Verify(amProxy, Org.Mockito.Mockito.Times(1)).GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest>()); }
public virtual void TestJobReportFromHistoryServer() { MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())). ThenReturn(GetJobReportResponseFromHistoryServer()); ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId ())).ThenReturn(null); ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy , rm); JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); NUnit.Framework.Assert.AreEqual("TestJobFilePath", jobStatus.GetJobFile()); NUnit.Framework.Assert.AreEqual("http://TestTrackingUrl", jobStatus.GetTrackingUrl ()); NUnit.Framework.Assert.AreEqual(1.0f, jobStatus.GetMapProgress(), 0.0f); NUnit.Framework.Assert.AreEqual(1.0f, jobStatus.GetReduceProgress(), 0.0f); }
public virtual void TestRemoteExceptionFromHistoryServer() { MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())). ThenThrow(new IOException("Job ID doesnot Exist")); ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId ())).ThenReturn(null); ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy , rm); try { clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.Fail("Invoke should throw exception after retries."); } catch (IOException e) { NUnit.Framework.Assert.IsTrue(e.Message.Contains("Job ID doesnot Exist")); } }
public ClientCache(Configuration conf, ResourceMgrDelegate rm) { this.conf = conf; this.rm = rm; }
/// <exception cref="System.Exception"/> public virtual void TestGetHSDelegationToken() { try { Configuration conf = new Configuration(); // Setup mock service IPEndPoint mockRmAddress = new IPEndPoint("localhost", 4444); Text rmTokenSevice = SecurityUtil.BuildTokenService(mockRmAddress); IPEndPoint mockHsAddress = new IPEndPoint("localhost", 9200); Text hsTokenSevice = SecurityUtil.BuildTokenService(mockHsAddress); // Setup mock rm token RMDelegationTokenIdentifier tokenIdentifier = new RMDelegationTokenIdentifier(new Text("owner"), new Text("renewer"), new Text("real")); Org.Apache.Hadoop.Security.Token.Token <RMDelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token <RMDelegationTokenIdentifier>(new byte[0], new byte[0], tokenIdentifier.GetKind( ), rmTokenSevice); token.SetKind(RMDelegationTokenIdentifier.KindName); // Setup mock history token Org.Apache.Hadoop.Yarn.Api.Records.Token historyToken = Org.Apache.Hadoop.Yarn.Api.Records.Token .NewInstance(new byte[0], MRDelegationTokenIdentifier.KindName.ToString(), new byte [0], hsTokenSevice.ToString()); GetDelegationTokenResponse getDtResponse = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetDelegationTokenResponse>(); getDtResponse.SetDelegationToken(historyToken); // mock services MRClientProtocol mockHsProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(); Org.Mockito.Mockito.DoReturn(mockHsAddress).When(mockHsProxy).GetConnectAddress(); Org.Mockito.Mockito.DoReturn(getDtResponse).When(mockHsProxy).GetDelegationToken( Matchers.Any <GetDelegationTokenRequest>()); ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); Org.Mockito.Mockito.DoReturn(rmTokenSevice).When(rmDelegate).GetRMDelegationTokenService (); ClientCache clientCache = Org.Mockito.Mockito.Mock <ClientCache>(); Org.Mockito.Mockito.DoReturn(mockHsProxy).When(clientCache).GetInitializedHSProxy (); Credentials creds = new Credentials(); YARNRunner yarnRunner = new YARNRunner(conf, rmDelegate, clientCache); // No HS token if no RM token yarnRunner.AddHistoryToken(creds); Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(0)).GetDelegationToken (Matchers.Any <GetDelegationTokenRequest>()); // No HS token if RM token, but secirity disabled. creds.AddToken(new Text("rmdt"), token); yarnRunner.AddHistoryToken(creds); Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(0)).GetDelegationToken (Matchers.Any <GetDelegationTokenRequest>()); conf.Set(CommonConfigurationKeys.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); creds = new Credentials(); // No HS token if no RM token, security enabled yarnRunner.AddHistoryToken(creds); Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(0)).GetDelegationToken (Matchers.Any <GetDelegationTokenRequest>()); // HS token if RM token present, security enabled creds.AddToken(new Text("rmdt"), token); yarnRunner.AddHistoryToken(creds); Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(1)).GetDelegationToken (Matchers.Any <GetDelegationTokenRequest>()); // No additional call to get HS token if RM and HS token present yarnRunner.AddHistoryToken(creds); Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(1)).GetDelegationToken (Matchers.Any <GetDelegationTokenRequest>()); } finally { // Back to defaults. UserGroupInformation.SetConfiguration(new Configuration()); } }
public _ClientServiceDelegate_205(MRClientProtocol amProxy, Configuration baseArg1 , ResourceMgrDelegate baseArg2, JobID baseArg3, MRClientProtocol baseArg4) : base(baseArg1, baseArg2, baseArg3, baseArg4) { this.amProxy = amProxy; }
/// <summary> /// Similar to /// <see cref="YARNRunner(Org.Apache.Hadoop.Conf.Configuration)"/> /// but allowing injecting /// <see cref="ResourceMgrDelegate"/> /// . Enables mocking and testing. /// </summary> /// <param name="conf">the configuration object for the client</param> /// <param name="resMgrDelegate">the resourcemanager client handle.</param> public YARNRunner(Configuration conf, ResourceMgrDelegate resMgrDelegate) : this(conf, resMgrDelegate, new ClientCache(conf, resMgrDelegate)) { }
public virtual void SetResourceMgrDelegate(ResourceMgrDelegate resMgrDelegate) { this.resMgrDelegate = resMgrDelegate; }
public virtual void TestReconnectOnAMRestart() { //test not applicable when AM not reachable //as instantiateAMProxy is not called at all if (!isAMReachableFromClient) { return; } MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); // RM returns AM1 url, null, null and AM2 url on invocations. // Nulls simulate the time when AM2 is in the process of restarting. ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); try { Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenReturn (GetRunningApplicationReport("am1", 78)).ThenReturn(GetRunningApplicationReport( null, 0)).ThenReturn(GetRunningApplicationReport(null, 0)).ThenReturn(GetRunningApplicationReport ("am2", 90)); } catch (YarnException e) { throw new IOException(e); } Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse jobReportResponse1 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse >(); Org.Mockito.Mockito.When(jobReportResponse1.GetJobReport()).ThenReturn(MRBuilderUtils .NewJobReport(jobId, "jobName-firstGen", "user", JobState.Running, 0, 0, 0, 0, 0 , 0, 0, "anything", null, false, string.Empty)); // First AM returns a report with jobName firstGen and simulates AM shutdown // on second invocation. MRClientProtocol firstGenAMProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(); Org.Mockito.Mockito.When(firstGenAMProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest >())).ThenReturn(jobReportResponse1).ThenThrow(new RuntimeException("AM is down!" )); Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse jobReportResponse2 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse >(); Org.Mockito.Mockito.When(jobReportResponse2.GetJobReport()).ThenReturn(MRBuilderUtils .NewJobReport(jobId, "jobName-secondGen", "user", JobState.Running, 0, 0, 0, 0, 0, 0, 0, "anything", null, false, string.Empty)); // Second AM generation returns a report with jobName secondGen MRClientProtocol secondGenAMProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(); Org.Mockito.Mockito.When(secondGenAMProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest >())).ThenReturn(jobReportResponse2); ClientServiceDelegate clientServiceDelegate = Org.Mockito.Mockito.Spy(GetClientServiceDelegate (historyServerProxy, rmDelegate)); // First time, connection should be to AM1, then to AM2. Further requests // should use the same proxy to AM2 and so instantiateProxy shouldn't be // called. Org.Mockito.Mockito.DoReturn(firstGenAMProxy).DoReturn(secondGenAMProxy).When(clientServiceDelegate ).InstantiateAMProxy(Matchers.Any <IPEndPoint>()); JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); NUnit.Framework.Assert.AreEqual("jobName-firstGen", jobStatus.GetJobName()); jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); NUnit.Framework.Assert.AreEqual("jobName-secondGen", jobStatus.GetJobName()); jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); NUnit.Framework.Assert.AreEqual("jobName-secondGen", jobStatus.GetJobName()); Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(2)).InstantiateAMProxy (Matchers.Any <IPEndPoint>()); }