public virtual void TestRetriesOnAMConnectionFailures()
{
if (!isAMReachableFromClient)
{
return;
}
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
())).ThenReturn(GetRunningApplicationReport("am1", 78));
// throw exception in 1st, 2nd, 3rd and 4th call of getJobReport, and
// succeed in the 5th call.
MRClientProtocol amProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>();
Org.Mockito.Mockito.When(amProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
>())).ThenThrow(new RuntimeException("11")).ThenThrow(new RuntimeException("22")
).ThenThrow(new RuntimeException("33")).ThenThrow(new RuntimeException("44")).ThenReturn
(GetJobReportResponse());
Configuration conf = new YarnConfiguration();
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient);
ClientServiceDelegate clientServiceDelegate = new _ClientServiceDelegate_167(amProxy
, conf, rm, oldJobId, null);
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
// assert maxClientRetry is not decremented.
NUnit.Framework.Assert.AreEqual(conf.GetInt(MRJobConfig.MrClientMaxRetries, MRJobConfig
.DefaultMrClientMaxRetries), clientServiceDelegate.GetMaxClientRetry());
Org.Mockito.Mockito.Verify(amProxy, Org.Mockito.Mockito.Times(5)).GetJobReport(Matchers.Any
<Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest>());
}
/// <exception cref="System.Exception"/>
private void TestMinimumAllocation(YarnConfiguration conf, int testAlloc)
{
MockRM rm = new MockRM(conf);
rm.Start();
// Register node1
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
// Submit an application
RMApp app1 = rm.SubmitApp(testAlloc);
// kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId());
am1.RegisterAppAttempt();
SchedulerNodeReport report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId
());
int checkAlloc = conf.GetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, YarnConfiguration
.DefaultRmSchedulerMinimumAllocationMb);
NUnit.Framework.Assert.AreEqual(checkAlloc, report_nm1.GetUsedResource().GetMemory
());
rm.Stop();
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
lock (this)
{
Configuration config = new YarnConfiguration(conf);
// create nm-local-dirs and configure them for the nodemanager
string localDirsString = this.PrepareDirs("local", this._enclosing.numLocalDirs);
config.Set(YarnConfiguration.NmLocalDirs, localDirsString);
// create nm-log-dirs and configure them for the nodemanager
string logDirsString = this.PrepareDirs("log", this._enclosing.numLogDirs);
config.Set(YarnConfiguration.NmLogDirs, logDirsString);
config.SetInt(YarnConfiguration.NmPmemMb, config.GetInt(YarnConfiguration.YarnMiniclusterNmPmemMb
, YarnConfiguration.DefaultYarnMiniclusterNmPmemMb));
config.Set(YarnConfiguration.NmAddress, MiniYARNCluster.GetHostname() + ":0");
config.Set(YarnConfiguration.NmLocalizerAddress, MiniYARNCluster.GetHostname() +
":0");
WebAppUtils.SetNMWebAppHostNameAndPort(config, MiniYARNCluster.GetHostname(), 0);
// Disable resource checks by default
if (!config.GetBoolean(YarnConfiguration.YarnMiniclusterControlResourceMonitoring
, YarnConfiguration.DefaultYarnMiniclusterControlResourceMonitoring))
{
config.SetBoolean(YarnConfiguration.NmPmemCheckEnabled, false);
config.SetBoolean(YarnConfiguration.NmVmemCheckEnabled, false);
}
MiniYARNCluster.Log.Info("Starting NM: " + this.index);
this._enclosing.nodeManagers[this.index].Init(config);
base.ServiceInit(config);
}
}
public virtual void TestRMDownForJobStatusBeforeGetAMReportWithRetryTimes()
{
Configuration conf = new YarnConfiguration();
conf.SetInt(MRJobConfig.MrClientMaxRetries, 2);
TestRMDownForJobStatusBeforeGetAMReport(conf, conf.GetInt(MRJobConfig.MrClientMaxRetries
, MRJobConfig.DefaultMrClientMaxRetries));
}
public virtual void Init(Configuration configuration, RMProxy <T> rmProxy, Type protocol
)
{
this.rmProxy = rmProxy;
this.protocol = protocol;
this.rmProxy.CheckAllowedProtocols(this.protocol);
this.conf = new YarnConfiguration(configuration);
ICollection <string> rmIds = HAUtil.GetRMHAIds(conf);
this.rmServiceIds = Sharpen.Collections.ToArray(rmIds, new string[rmIds.Count]);
conf.Set(YarnConfiguration.RmHaId, rmServiceIds[currentProxyIndex]);
conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectMaxRetriesKey, conf.GetInt
(YarnConfiguration.ClientFailoverRetries, YarnConfiguration.DefaultClientFailoverRetries
));
conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectMaxRetriesOnSocketTimeoutsKey
, conf.GetInt(YarnConfiguration.ClientFailoverRetriesOnSocketTimeouts, YarnConfiguration
.DefaultClientFailoverRetriesOnSocketTimeouts));
}
// Couldn't find an Active RM
/// <exception cref="System.Exception"/>
private static HAServiceProtocol.HAServiceState GetHAState(YarnConfiguration yarnConf
)
{
HAServiceTarget haServiceTarget;
int rpcTimeoutForChecks = yarnConf.GetInt(CommonConfigurationKeys.HaFcCliCheckTimeoutKey
, CommonConfigurationKeys.HaFcCliCheckTimeoutDefault);
yarnConf.Set(CommonConfigurationKeys.HadoopSecurityServiceUserNameKey, yarnConf.Get
(YarnConfiguration.RmPrincipal, string.Empty));
haServiceTarget = new RMHAServiceTarget(yarnConf);
HAServiceProtocol proto = haServiceTarget.GetProxy(yarnConf, rpcTimeoutForChecks);
HAServiceProtocol.HAServiceState haState = proto.GetServiceStatus().GetState();
return(haState);
}
public virtual void TestNoRetryOnAMAuthorizationException()
{
if (!isAMReachableFromClient)
{
return;
}
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
())).ThenReturn(GetRunningApplicationReport("am1", 78));
// throw authorization exception on first invocation
MRClientProtocol amProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>();
Org.Mockito.Mockito.When(amProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
>())).ThenThrow(new AuthorizationException("Denied"));
Configuration conf = new YarnConfiguration();
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient);
ClientServiceDelegate clientServiceDelegate = new _ClientServiceDelegate_205(amProxy
, conf, rm, oldJobId, null);
try
{
clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.Fail("Exception should be thrown upon AuthorizationException"
);
}
catch (IOException e)
{
NUnit.Framework.Assert.AreEqual(typeof(AuthorizationException).FullName + ": Denied"
, e.Message);
}
// assert maxClientRetry is not decremented.
NUnit.Framework.Assert.AreEqual(conf.GetInt(MRJobConfig.MrClientMaxRetries, MRJobConfig
.DefaultMrClientMaxRetries), clientServiceDelegate.GetMaxClientRetry());
Org.Mockito.Mockito.Verify(amProxy, Org.Mockito.Mockito.Times(1)).GetJobReport(Matchers.Any
<Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest>());
}
