public virtual void TestRMDownRestoreForJobStatusBeforeGetAMReport()
{
Configuration conf = new YarnConfiguration();
conf.SetInt(MRJobConfig.MrClientMaxRetries, 3);
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient);
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
>())).ThenReturn(GetJobReportResponse());
ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
try
{
Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenThrow
(new UndeclaredThrowableException(new IOException("Connection refuced1"))).ThenThrow
(new UndeclaredThrowableException(new IOException("Connection refuced2"))).ThenReturn
(GetFinishedApplicationReport());
ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rmDelegate
, oldJobId, historyServerProxy);
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
Org.Mockito.Mockito.Verify(rmDelegate, Org.Mockito.Mockito.Times(3)).GetApplicationReport
(Matchers.Any <ApplicationId>());
NUnit.Framework.Assert.IsNotNull(jobStatus);
}
catch (YarnException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
private void TestRMDownForJobStatusBeforeGetAMReport(Configuration conf, int noOfRetries
)
{
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient);
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
try
{
Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenThrow
(new UndeclaredThrowableException(new IOException("Connection refuced1"))).ThenThrow
(new UndeclaredThrowableException(new IOException("Connection refuced2"))).ThenThrow
(new UndeclaredThrowableException(new IOException("Connection refuced3")));
ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rmDelegate
, oldJobId, historyServerProxy);
try
{
clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.Fail("It should throw exception after retries");
}
catch (IOException e)
{
System.Console.Out.WriteLine("fail to get job status,and e=" + e.ToString());
}
Org.Mockito.Mockito.Verify(rmDelegate, Org.Mockito.Mockito.Times(noOfRetries)).GetApplicationReport
(Matchers.Any <ApplicationId>());
}
catch (YarnException e)
{
throw new IOException(e);
}
}
public virtual void TestRetriesOnAMConnectionFailures()
{
if (!isAMReachableFromClient)
{
return;
}
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
())).ThenReturn(GetRunningApplicationReport("am1", 78));
// throw exception in 1st, 2nd, 3rd and 4th call of getJobReport, and
// succeed in the 5th call.
MRClientProtocol amProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>();
Org.Mockito.Mockito.When(amProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
>())).ThenThrow(new RuntimeException("11")).ThenThrow(new RuntimeException("22")
).ThenThrow(new RuntimeException("33")).ThenThrow(new RuntimeException("44")).ThenReturn
(GetJobReportResponse());
Configuration conf = new YarnConfiguration();
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient);
ClientServiceDelegate clientServiceDelegate = new _ClientServiceDelegate_167(amProxy
, conf, rm, oldJobId, null);
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
// assert maxClientRetry is not decremented.
NUnit.Framework.Assert.AreEqual(conf.GetInt(MRJobConfig.MrClientMaxRetries, MRJobConfig
.DefaultMrClientMaxRetries), clientServiceDelegate.GetMaxClientRetry());
Org.Mockito.Mockito.Verify(amProxy, Org.Mockito.Mockito.Times(5)).GetJobReport(Matchers.Any
<Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest>());
}
public virtual void TestAMAccessDisabled()
{
//test only applicable when AM not reachable
if (isAMReachableFromClient)
{
return;
}
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())).
ThenReturn(GetJobReportResponseFromHistoryServer());
ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
try
{
Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenReturn
(GetRunningApplicationReport("am1", 78)).ThenReturn(GetRunningApplicationReport(
"am1", 78)).ThenReturn(GetRunningApplicationReport("am1", 78)).ThenReturn(GetFinishedApplicationReport
());
}
catch (YarnException e)
{
throw new IOException(e);
}
ClientServiceDelegate clientServiceDelegate = Org.Mockito.Mockito.Spy(GetClientServiceDelegate
(historyServerProxy, rmDelegate));
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetJobName());
Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy
(Matchers.Any <IPEndPoint>());
// Should not reach AM even for second and third times too.
jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetJobName());
Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy
(Matchers.Any <IPEndPoint>());
jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetJobName());
Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy
(Matchers.Any <IPEndPoint>());
// The third time around, app is completed, so should go to JHS
JobStatus jobStatus1 = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus1);
NUnit.Framework.Assert.AreEqual("TestJobFilePath", jobStatus1.GetJobFile());
NUnit.Framework.Assert.AreEqual("http://TestTrackingUrl", jobStatus1.GetTrackingUrl
());
NUnit.Framework.Assert.AreEqual(1.0f, jobStatus1.GetMapProgress(), 0.0f);
NUnit.Framework.Assert.AreEqual(1.0f, jobStatus1.GetReduceProgress(), 0.0f);
Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy
(Matchers.Any <IPEndPoint>());
}
private ClientServiceDelegate GetClientServiceDelegate(MRClientProtocol historyServerProxy
, ResourceMgrDelegate rm)
{
Configuration conf = new YarnConfiguration();
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient);
ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rm,
oldJobId, historyServerProxy);
return(clientServiceDelegate);
}
/// <summary>
/// Similar to
/// <see cref="YARNRunner(Org.Apache.Hadoop.Conf.Configuration, ResourceMgrDelegate)"
/// />
/// but allowing injecting
/// <see cref="ClientCache"/>
/// . Enable mocking and testing.
/// </summary>
/// <param name="conf">the configuration object</param>
/// <param name="resMgrDelegate">the resource manager delegate</param>
/// <param name="clientCache">the client cache object.</param>
public YARNRunner(Configuration conf, ResourceMgrDelegate resMgrDelegate, ClientCache
clientCache)
{
this.conf = conf;
try
{
this.resMgrDelegate = resMgrDelegate;
this.clientCache = clientCache;
this.defaultFileContext = FileContext.GetFileContext(this.conf);
}
catch (UnsupportedFileSystemException ufe)
{
throw new RuntimeException("Error in instantiating YarnClient", ufe);
}
}
/// <exception cref="System.IO.IOException"/>
private ResourceMgrDelegate GetRMDelegate()
{
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
try
{
ApplicationId appId = jobId.GetAppId();
Org.Mockito.Mockito.When(rm.GetApplicationReport(appId)).ThenThrow(new ApplicationNotFoundException
(appId + " not found"));
}
catch (YarnException e)
{
throw new IOException(e);
}
return(rm);
}
public ClientServiceDelegate(Configuration conf, ResourceMgrDelegate rm, JobID jobId
, MRClientProtocol historyServerProxy)
{
// Caches for per-user NotRunningJobs
this.conf = new Configuration(conf);
// Cloning for modifying.
// For faster redirects from AM to HS.
this.conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectMaxRetriesKey, this
.conf.GetInt(MRJobConfig.MrClientToAmIpcMaxRetries, MRJobConfig.DefaultMrClientToAmIpcMaxRetries
));
this.conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectMaxRetriesOnSocketTimeoutsKey
, this.conf.GetInt(MRJobConfig.MrClientToAmIpcMaxRetriesOnTimeouts, MRJobConfig.
DefaultMrClientToAmIpcMaxRetriesOnTimeouts));
this.rm = rm;
this.jobId = jobId;
this.historyServerProxy = historyServerProxy;
this.appId = TypeConverter.ToYarn(jobId).GetAppId();
notRunningJobs = new Dictionary <JobState, Dictionary <string, NotRunningJob> >();
}
public virtual void TestCountersFromHistoryServer()
{
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
Org.Mockito.Mockito.When(historyServerProxy.GetCounters(GetCountersRequest())).ThenReturn
(GetCountersResponseFromHistoryServer());
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
())).ThenReturn(null);
ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy
, rm);
Counters counters = TypeConverter.ToYarn(clientServiceDelegate.GetJobCounters(oldJobId
));
NUnit.Framework.Assert.IsNotNull(counters);
NUnit.Framework.Assert.AreEqual(1001, counters.GetCounterGroup("dummyCounters").GetCounter
("dummyCounter").GetValue());
}
public virtual void TestRetriesOnConnectionFailure()
{
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())).
ThenThrow(new RuntimeException("1")).ThenThrow(new RuntimeException("2")).ThenReturn
(GetJobReportResponse());
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
())).ThenReturn(null);
ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy
, rm);
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
Org.Mockito.Mockito.Verify(historyServerProxy, Org.Mockito.Mockito.Times(3)).GetJobReport
(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
>());
}
public virtual void TestHistoryServerNotConfigured()
{
//RM doesn't have app report and job History Server is not configured
ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(null, GetRMDelegate
());
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetUsername());
NUnit.Framework.Assert.AreEqual(JobStatus.State.Prep, jobStatus.GetState());
//RM has app report and job History Server is not configured
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
ApplicationReport applicationReport = GetFinishedApplicationReport();
Org.Mockito.Mockito.When(rm.GetApplicationReport(jobId.GetAppId())).ThenReturn(applicationReport
);
clientServiceDelegate = GetClientServiceDelegate(null, rm);
jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.AreEqual(applicationReport.GetUser(), jobStatus.GetUsername
());
NUnit.Framework.Assert.AreEqual(JobStatus.State.Succeeded, jobStatus.GetState());
}
protected override void SetUp()
{
resourceMgrDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
conf = new YarnConfiguration();
conf.Set(YarnConfiguration.RmPrincipal, "mapred/host@REALM");
clientCache = new ClientCache(conf, resourceMgrDelegate);
clientCache = Org.Mockito.Mockito.Spy(clientCache);
yarnRunner = new YARNRunner(conf, resourceMgrDelegate, clientCache);
yarnRunner = Org.Mockito.Mockito.Spy(yarnRunner);
submissionContext = Org.Mockito.Mockito.Mock <ApplicationSubmissionContext>();
Org.Mockito.Mockito.DoAnswer(new _Answer_146(this)).When(yarnRunner).CreateApplicationSubmissionContext
(Matchers.Any <Configuration>(), Matchers.Any <string>(), Matchers.Any <Credentials
>());
appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 1);
jobId = TypeConverter.FromYarn(appId);
if (testWorkDir.Exists())
{
FileContext.GetLocalFSFileContext().Delete(new Path(testWorkDir.ToString()), true
);
}
testWorkDir.Mkdirs();
}
public virtual void TestNoRetryOnAMAuthorizationException()
{
if (!isAMReachableFromClient)
{
return;
}
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
())).ThenReturn(GetRunningApplicationReport("am1", 78));
// throw authorization exception on first invocation
MRClientProtocol amProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>();
Org.Mockito.Mockito.When(amProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
>())).ThenThrow(new AuthorizationException("Denied"));
Configuration conf = new YarnConfiguration();
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient);
ClientServiceDelegate clientServiceDelegate = new _ClientServiceDelegate_205(amProxy
, conf, rm, oldJobId, null);
try
{
clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.Fail("Exception should be thrown upon AuthorizationException"
);
}
catch (IOException e)
{
NUnit.Framework.Assert.AreEqual(typeof(AuthorizationException).FullName + ": Denied"
, e.Message);
}
// assert maxClientRetry is not decremented.
NUnit.Framework.Assert.AreEqual(conf.GetInt(MRJobConfig.MrClientMaxRetries, MRJobConfig
.DefaultMrClientMaxRetries), clientServiceDelegate.GetMaxClientRetry());
Org.Mockito.Mockito.Verify(amProxy, Org.Mockito.Mockito.Times(1)).GetJobReport(Matchers.Any
<Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest>());
}
public virtual void TestJobReportFromHistoryServer()
{
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())).
ThenReturn(GetJobReportResponseFromHistoryServer());
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
())).ThenReturn(null);
ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy
, rm);
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
NUnit.Framework.Assert.AreEqual("TestJobFilePath", jobStatus.GetJobFile());
NUnit.Framework.Assert.AreEqual("http://TestTrackingUrl", jobStatus.GetTrackingUrl
());
NUnit.Framework.Assert.AreEqual(1.0f, jobStatus.GetMapProgress(), 0.0f);
NUnit.Framework.Assert.AreEqual(1.0f, jobStatus.GetReduceProgress(), 0.0f);
}
public virtual void TestRemoteExceptionFromHistoryServer()
{
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())).
ThenThrow(new IOException("Job ID doesnot Exist"));
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
())).ThenReturn(null);
ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy
, rm);
try
{
clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.Fail("Invoke should throw exception after retries.");
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("Job ID doesnot Exist"));
}
}
public ClientCache(Configuration conf, ResourceMgrDelegate rm)
{
this.conf = conf;
this.rm = rm;
}
/// <exception cref="System.Exception"/>
public virtual void TestGetHSDelegationToken()
{
try
{
Configuration conf = new Configuration();
// Setup mock service
IPEndPoint mockRmAddress = new IPEndPoint("localhost", 4444);
Text rmTokenSevice = SecurityUtil.BuildTokenService(mockRmAddress);
IPEndPoint mockHsAddress = new IPEndPoint("localhost", 9200);
Text hsTokenSevice = SecurityUtil.BuildTokenService(mockHsAddress);
// Setup mock rm token
RMDelegationTokenIdentifier tokenIdentifier = new RMDelegationTokenIdentifier(new
Text("owner"), new Text("renewer"), new Text("real"));
Org.Apache.Hadoop.Security.Token.Token <RMDelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<RMDelegationTokenIdentifier>(new byte[0], new byte[0], tokenIdentifier.GetKind(
), rmTokenSevice);
token.SetKind(RMDelegationTokenIdentifier.KindName);
// Setup mock history token
Org.Apache.Hadoop.Yarn.Api.Records.Token historyToken = Org.Apache.Hadoop.Yarn.Api.Records.Token
.NewInstance(new byte[0], MRDelegationTokenIdentifier.KindName.ToString(), new byte
[0], hsTokenSevice.ToString());
GetDelegationTokenResponse getDtResponse = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<GetDelegationTokenResponse>();
getDtResponse.SetDelegationToken(historyToken);
// mock services
MRClientProtocol mockHsProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>();
Org.Mockito.Mockito.DoReturn(mockHsAddress).When(mockHsProxy).GetConnectAddress();
Org.Mockito.Mockito.DoReturn(getDtResponse).When(mockHsProxy).GetDelegationToken(
Matchers.Any <GetDelegationTokenRequest>());
ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
Org.Mockito.Mockito.DoReturn(rmTokenSevice).When(rmDelegate).GetRMDelegationTokenService
();
ClientCache clientCache = Org.Mockito.Mockito.Mock <ClientCache>();
Org.Mockito.Mockito.DoReturn(mockHsProxy).When(clientCache).GetInitializedHSProxy
();
Credentials creds = new Credentials();
YARNRunner yarnRunner = new YARNRunner(conf, rmDelegate, clientCache);
// No HS token if no RM token
yarnRunner.AddHistoryToken(creds);
Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(0)).GetDelegationToken
(Matchers.Any <GetDelegationTokenRequest>());
// No HS token if RM token, but secirity disabled.
creds.AddToken(new Text("rmdt"), token);
yarnRunner.AddHistoryToken(creds);
Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(0)).GetDelegationToken
(Matchers.Any <GetDelegationTokenRequest>());
conf.Set(CommonConfigurationKeys.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
creds = new Credentials();
// No HS token if no RM token, security enabled
yarnRunner.AddHistoryToken(creds);
Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(0)).GetDelegationToken
(Matchers.Any <GetDelegationTokenRequest>());
// HS token if RM token present, security enabled
creds.AddToken(new Text("rmdt"), token);
yarnRunner.AddHistoryToken(creds);
Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(1)).GetDelegationToken
(Matchers.Any <GetDelegationTokenRequest>());
// No additional call to get HS token if RM and HS token present
yarnRunner.AddHistoryToken(creds);
Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(1)).GetDelegationToken
(Matchers.Any <GetDelegationTokenRequest>());
}
finally
{
// Back to defaults.
UserGroupInformation.SetConfiguration(new Configuration());
}
}
public _ClientServiceDelegate_205(MRClientProtocol amProxy, Configuration baseArg1
, ResourceMgrDelegate baseArg2, JobID baseArg3, MRClientProtocol baseArg4)
: base(baseArg1, baseArg2, baseArg3, baseArg4)
{
this.amProxy = amProxy;
}
/// <summary>
/// Similar to
/// <see cref="YARNRunner(Org.Apache.Hadoop.Conf.Configuration)"/>
/// but allowing injecting
/// <see cref="ResourceMgrDelegate"/>
/// . Enables mocking and testing.
/// </summary>
/// <param name="conf">the configuration object for the client</param>
/// <param name="resMgrDelegate">the resourcemanager client handle.</param>
public YARNRunner(Configuration conf, ResourceMgrDelegate resMgrDelegate)
: this(conf, resMgrDelegate, new ClientCache(conf, resMgrDelegate))
{
}
public virtual void SetResourceMgrDelegate(ResourceMgrDelegate resMgrDelegate)
{
this.resMgrDelegate = resMgrDelegate;
}
public virtual void TestReconnectOnAMRestart()
{
//test not applicable when AM not reachable
//as instantiateAMProxy is not called at all
if (!isAMReachableFromClient)
{
return;
}
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
// RM returns AM1 url, null, null and AM2 url on invocations.
// Nulls simulate the time when AM2 is in the process of restarting.
ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
try
{
Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenReturn
(GetRunningApplicationReport("am1", 78)).ThenReturn(GetRunningApplicationReport(
null, 0)).ThenReturn(GetRunningApplicationReport(null, 0)).ThenReturn(GetRunningApplicationReport
("am2", 90));
}
catch (YarnException e)
{
throw new IOException(e);
}
Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse jobReportResponse1
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse
>();
Org.Mockito.Mockito.When(jobReportResponse1.GetJobReport()).ThenReturn(MRBuilderUtils
.NewJobReport(jobId, "jobName-firstGen", "user", JobState.Running, 0, 0, 0, 0, 0
, 0, 0, "anything", null, false, string.Empty));
// First AM returns a report with jobName firstGen and simulates AM shutdown
// on second invocation.
MRClientProtocol firstGenAMProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>();
Org.Mockito.Mockito.When(firstGenAMProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
>())).ThenReturn(jobReportResponse1).ThenThrow(new RuntimeException("AM is down!"
));
Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse jobReportResponse2
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse
>();
Org.Mockito.Mockito.When(jobReportResponse2.GetJobReport()).ThenReturn(MRBuilderUtils
.NewJobReport(jobId, "jobName-secondGen", "user", JobState.Running, 0, 0, 0, 0,
0, 0, 0, "anything", null, false, string.Empty));
// Second AM generation returns a report with jobName secondGen
MRClientProtocol secondGenAMProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>();
Org.Mockito.Mockito.When(secondGenAMProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
>())).ThenReturn(jobReportResponse2);
ClientServiceDelegate clientServiceDelegate = Org.Mockito.Mockito.Spy(GetClientServiceDelegate
(historyServerProxy, rmDelegate));
// First time, connection should be to AM1, then to AM2. Further requests
// should use the same proxy to AM2 and so instantiateProxy shouldn't be
// called.
Org.Mockito.Mockito.DoReturn(firstGenAMProxy).DoReturn(secondGenAMProxy).When(clientServiceDelegate
).InstantiateAMProxy(Matchers.Any <IPEndPoint>());
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
NUnit.Framework.Assert.AreEqual("jobName-firstGen", jobStatus.GetJobName());
jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
NUnit.Framework.Assert.AreEqual("jobName-secondGen", jobStatus.GetJobName());
jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
NUnit.Framework.Assert.AreEqual("jobName-secondGen", jobStatus.GetJobName());
Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(2)).InstantiateAMProxy
(Matchers.Any <IPEndPoint>());
}
