/// <exception cref="System.IO.IOException"/>
private void TestRMDownForJobStatusBeforeGetAMReport(Configuration conf, int noOfRetries
)
{
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient);
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
try
{
Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenThrow
(new UndeclaredThrowableException(new IOException("Connection refuced1"))).ThenThrow
(new UndeclaredThrowableException(new IOException("Connection refuced2"))).ThenThrow
(new UndeclaredThrowableException(new IOException("Connection refuced3")));
ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rmDelegate
, oldJobId, historyServerProxy);
try
{
clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.Fail("It should throw exception after retries");
}
catch (IOException e)
{
System.Console.Out.WriteLine("fail to get job status,and e=" + e.ToString());
}
Org.Mockito.Mockito.Verify(rmDelegate, Org.Mockito.Mockito.Times(noOfRetries)).GetApplicationReport
(Matchers.Any <ApplicationId>());
}
catch (YarnException e)
{
throw new IOException(e);
}
}
//TODO: evict from the cache on some threshold
public virtual ClientServiceDelegate GetClient(JobID jobId)
{
lock (this)
{
if (hsProxy == null)
{
try
{
hsProxy = InstantiateHistoryProxy();
}
catch (IOException e)
{
Log.Warn("Could not connect to History server.", e);
throw new YarnRuntimeException("Could not connect to History server.", e);
}
}
ClientServiceDelegate client = cache[jobId];
if (client == null)
{
client = new ClientServiceDelegate(conf, rm, jobId, hsProxy);
cache[jobId] = client;
}
return(client);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestJobKill()
{
clientDelegate = Org.Mockito.Mockito.Mock <ClientServiceDelegate>();
Org.Mockito.Mockito.When(clientDelegate.GetJobStatus(Matchers.Any <JobID>())).ThenReturn
(new JobStatus(jobId, 0f, 0f, 0f, 0f, JobStatus.State.Prep, JobPriority.High, "tmp"
, "tmp", "tmp", "tmp"));
Org.Mockito.Mockito.When(clientDelegate.KillJob(Matchers.Any <JobID>())).ThenReturn
(true);
Org.Mockito.Mockito.DoAnswer(new _Answer_177(this)).When(clientCache).GetClient(Matchers.Any
<JobID>());
yarnRunner.KillJob(jobId);
Org.Mockito.Mockito.Verify(resourceMgrDelegate).KillApplication(appId);
Org.Mockito.Mockito.When(clientDelegate.GetJobStatus(Matchers.Any <JobID>())).ThenReturn
(new JobStatus(jobId, 0f, 0f, 0f, 0f, JobStatus.State.Running, JobPriority.High,
"tmp", "tmp", "tmp", "tmp"));
yarnRunner.KillJob(jobId);
Org.Mockito.Mockito.Verify(clientDelegate).KillJob(jobId);
Org.Mockito.Mockito.When(clientDelegate.GetJobStatus(Matchers.Any <JobID>())).ThenReturn
(null);
Org.Mockito.Mockito.When(resourceMgrDelegate.GetApplicationReport(Matchers.Any <ApplicationId
>())).ThenReturn(ApplicationReport.NewInstance(appId, null, "tmp", "tmp", "tmp",
"tmp", 0, null, YarnApplicationState.Finished, "tmp", "tmp", 0l, 0l, FinalApplicationStatus
.Succeeded, null, null, 0f, "tmp", null));
yarnRunner.KillJob(jobId);
Org.Mockito.Mockito.Verify(clientDelegate).KillJob(jobId);
}
public virtual void TestRMDownRestoreForJobStatusBeforeGetAMReport()
{
Configuration conf = new YarnConfiguration();
conf.SetInt(MRJobConfig.MrClientMaxRetries, 3);
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient);
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
>())).ThenReturn(GetJobReportResponse());
ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
try
{
Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenThrow
(new UndeclaredThrowableException(new IOException("Connection refuced1"))).ThenThrow
(new UndeclaredThrowableException(new IOException("Connection refuced2"))).ThenReturn
(GetFinishedApplicationReport());
ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rmDelegate
, oldJobId, historyServerProxy);
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
Org.Mockito.Mockito.Verify(rmDelegate, Org.Mockito.Mockito.Times(3)).GetApplicationReport
(Matchers.Any <ApplicationId>());
NUnit.Framework.Assert.IsNotNull(jobStatus);
}
catch (YarnException e)
{
throw new IOException(e);
}
}
public virtual void TestAMAccessDisabled()
{
//test only applicable when AM not reachable
if (isAMReachableFromClient)
{
return;
}
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())).
ThenReturn(GetJobReportResponseFromHistoryServer());
ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
try
{
Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenReturn
(GetRunningApplicationReport("am1", 78)).ThenReturn(GetRunningApplicationReport(
"am1", 78)).ThenReturn(GetRunningApplicationReport("am1", 78)).ThenReturn(GetFinishedApplicationReport
());
}
catch (YarnException e)
{
throw new IOException(e);
}
ClientServiceDelegate clientServiceDelegate = Org.Mockito.Mockito.Spy(GetClientServiceDelegate
(historyServerProxy, rmDelegate));
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetJobName());
Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy
(Matchers.Any <IPEndPoint>());
// Should not reach AM even for second and third times too.
jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetJobName());
Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy
(Matchers.Any <IPEndPoint>());
jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetJobName());
Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy
(Matchers.Any <IPEndPoint>());
// The third time around, app is completed, so should go to JHS
JobStatus jobStatus1 = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus1);
NUnit.Framework.Assert.AreEqual("TestJobFilePath", jobStatus1.GetJobFile());
NUnit.Framework.Assert.AreEqual("http://TestTrackingUrl", jobStatus1.GetTrackingUrl
());
NUnit.Framework.Assert.AreEqual(1.0f, jobStatus1.GetMapProgress(), 0.0f);
NUnit.Framework.Assert.AreEqual(1.0f, jobStatus1.GetReduceProgress(), 0.0f);
Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy
(Matchers.Any <IPEndPoint>());
}
private ClientServiceDelegate GetClientServiceDelegate(MRClientProtocol historyServerProxy
, ResourceMgrDelegate rm)
{
Configuration conf = new YarnConfiguration();
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient);
ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rm,
oldJobId, historyServerProxy);
return(clientServiceDelegate);
}
public virtual void TestUnknownAppInRM()
{
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())).
ThenReturn(GetJobReportResponse());
ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy
, GetRMDelegate());
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
}
/// <exception cref="System.Exception"/>
public virtual void TestJobKillTimeout()
{
long timeToWaitBeforeHardKill = 10000 + MRJobConfig.DefaultMrAmHardKillTimeoutMs;
conf.SetLong(MRJobConfig.MrAmHardKillTimeoutMs, timeToWaitBeforeHardKill);
clientDelegate = Org.Mockito.Mockito.Mock <ClientServiceDelegate>();
Org.Mockito.Mockito.DoAnswer(new _Answer_212(this)).When(clientCache).GetClient(Matchers.Any
<JobID>());
Org.Mockito.Mockito.When(clientDelegate.GetJobStatus(Matchers.Any <JobID>())).ThenReturn
(new JobStatus(jobId, 0f, 0f, 0f, 0f, JobStatus.State.Running, JobPriority.High,
"tmp", "tmp", "tmp", "tmp"));
long startTimeMillis = Runtime.CurrentTimeMillis();
yarnRunner.KillJob(jobId);
NUnit.Framework.Assert.IsTrue("killJob should have waited at least " + timeToWaitBeforeHardKill
+ " ms.", Runtime.CurrentTimeMillis() - startTimeMillis >= timeToWaitBeforeHardKill
);
}
public virtual void TestCountersFromHistoryServer()
{
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
Org.Mockito.Mockito.When(historyServerProxy.GetCounters(GetCountersRequest())).ThenReturn
(GetCountersResponseFromHistoryServer());
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
())).ThenReturn(null);
ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy
, rm);
Counters counters = TypeConverter.ToYarn(clientServiceDelegate.GetJobCounters(oldJobId
));
NUnit.Framework.Assert.IsNotNull(counters);
NUnit.Framework.Assert.AreEqual(1001, counters.GetCounterGroup("dummyCounters").GetCounter
("dummyCounter").GetValue());
}
public virtual void TestRetriesOnConnectionFailure()
{
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())).
ThenThrow(new RuntimeException("1")).ThenThrow(new RuntimeException("2")).ThenReturn
(GetJobReportResponse());
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
())).ThenReturn(null);
ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy
, rm);
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
Org.Mockito.Mockito.Verify(historyServerProxy, Org.Mockito.Mockito.Times(3)).GetJobReport
(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
>());
}
public virtual void TestHistoryServerNotConfigured()
{
//RM doesn't have app report and job History Server is not configured
ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(null, GetRMDelegate
());
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetUsername());
NUnit.Framework.Assert.AreEqual(JobStatus.State.Prep, jobStatus.GetState());
//RM has app report and job History Server is not configured
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
ApplicationReport applicationReport = GetFinishedApplicationReport();
Org.Mockito.Mockito.When(rm.GetApplicationReport(jobId.GetAppId())).ThenReturn(applicationReport
);
clientServiceDelegate = GetClientServiceDelegate(null, rm);
jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.AreEqual(applicationReport.GetUser(), jobStatus.GetUsername
());
NUnit.Framework.Assert.AreEqual(JobStatus.State.Succeeded, jobStatus.GetState());
}
public virtual void TestJobReportFromHistoryServer()
{
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())).
ThenReturn(GetJobReportResponseFromHistoryServer());
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
())).ThenReturn(null);
ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy
, rm);
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
NUnit.Framework.Assert.AreEqual("TestJobFilePath", jobStatus.GetJobFile());
NUnit.Framework.Assert.AreEqual("http://TestTrackingUrl", jobStatus.GetTrackingUrl
());
NUnit.Framework.Assert.AreEqual(1.0f, jobStatus.GetMapProgress(), 0.0f);
NUnit.Framework.Assert.AreEqual(1.0f, jobStatus.GetReduceProgress(), 0.0f);
}
public virtual void TestRemoteExceptionFromHistoryServer()
{
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())).
ThenThrow(new IOException("Job ID doesnot Exist"));
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
())).ThenReturn(null);
ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy
, rm);
try
{
clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.Fail("Invoke should throw exception after retries.");
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("Job ID doesnot Exist"));
}
}
public _PrivilegedExceptionAction_202(ClientServiceDelegate _enclosing, IPEndPoint
finalServiceAddr)
{
this._enclosing = _enclosing;
this.finalServiceAddr = finalServiceAddr;
}
public virtual void TestReconnectOnAMRestart()
{
//test not applicable when AM not reachable
//as instantiateAMProxy is not called at all
if (!isAMReachableFromClient)
{
return;
}
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
// RM returns AM1 url, null, null and AM2 url on invocations.
// Nulls simulate the time when AM2 is in the process of restarting.
ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
try
{
Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenReturn
(GetRunningApplicationReport("am1", 78)).ThenReturn(GetRunningApplicationReport(
null, 0)).ThenReturn(GetRunningApplicationReport(null, 0)).ThenReturn(GetRunningApplicationReport
("am2", 90));
}
catch (YarnException e)
{
throw new IOException(e);
}
Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse jobReportResponse1
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse
>();
Org.Mockito.Mockito.When(jobReportResponse1.GetJobReport()).ThenReturn(MRBuilderUtils
.NewJobReport(jobId, "jobName-firstGen", "user", JobState.Running, 0, 0, 0, 0, 0
, 0, 0, "anything", null, false, string.Empty));
// First AM returns a report with jobName firstGen and simulates AM shutdown
// on second invocation.
MRClientProtocol firstGenAMProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>();
Org.Mockito.Mockito.When(firstGenAMProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
>())).ThenReturn(jobReportResponse1).ThenThrow(new RuntimeException("AM is down!"
));
Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse jobReportResponse2
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse
>();
Org.Mockito.Mockito.When(jobReportResponse2.GetJobReport()).ThenReturn(MRBuilderUtils
.NewJobReport(jobId, "jobName-secondGen", "user", JobState.Running, 0, 0, 0, 0,
0, 0, 0, "anything", null, false, string.Empty));
// Second AM generation returns a report with jobName secondGen
MRClientProtocol secondGenAMProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>();
Org.Mockito.Mockito.When(secondGenAMProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
>())).ThenReturn(jobReportResponse2);
ClientServiceDelegate clientServiceDelegate = Org.Mockito.Mockito.Spy(GetClientServiceDelegate
(historyServerProxy, rmDelegate));
// First time, connection should be to AM1, then to AM2. Further requests
// should use the same proxy to AM2 and so instantiateProxy shouldn't be
// called.
Org.Mockito.Mockito.DoReturn(firstGenAMProxy).DoReturn(secondGenAMProxy).When(clientServiceDelegate
).InstantiateAMProxy(Matchers.Any <IPEndPoint>());
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
NUnit.Framework.Assert.AreEqual("jobName-firstGen", jobStatus.GetJobName());
jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
NUnit.Framework.Assert.AreEqual("jobName-secondGen", jobStatus.GetJobName());
jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
NUnit.Framework.Assert.AreEqual("jobName-secondGen", jobStatus.GetJobName());
Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(2)).InstantiateAMProxy
(Matchers.Any <IPEndPoint>());
}
