public ContainerManagementProtocolProxy(Configuration conf, NMTokenCache nmTokenCache
)
{
this.conf = new Configuration(conf);
this.nmTokenCache = nmTokenCache;
maxConnectedNMs = conf.GetInt(YarnConfiguration.NmClientMaxNmProxies, YarnConfiguration
.DefaultNmClientMaxNmProxies);
if (maxConnectedNMs < 0)
{
throw new YarnRuntimeException(YarnConfiguration.NmClientMaxNmProxies + " (" + maxConnectedNMs
+ ") can not be less than 0.");
}
Log.Info(YarnConfiguration.NmClientMaxNmProxies + " : " + maxConnectedNMs);
if (maxConnectedNMs > 0)
{
cmProxy = new LinkedHashMap <string, ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData
>();
}
else
{
cmProxy = Sharpen.Collections.EmptyMap();
// Connections are not being cached so ensure connections close quickly
// to avoid creating thousands of RPC client threads on large clusters.
this.conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey,
0);
}
rpc = YarnRPC.Create(conf);
}
/// <exception cref="System.Exception"/>
public MRClientProtocol Run()
{
YarnRPC rpc = YarnRPC.Create(conf);
return((MRClientProtocol)rpc.GetProxy(typeof(MRClientProtocol), app.clientService
.GetBindAddress(), conf));
}
protected internal virtual ClientSCMProtocol CreateClientProxy()
{
YarnRPC rpc = YarnRPC.Create(GetConfig());
return((ClientSCMProtocol)rpc.GetProxy(typeof(ClientSCMProtocol), this.scmAddress
, GetConfig()));
}
public virtual void TestUnknownCall()
{
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.IpcRpcImpl, typeof(HadoopYarnProtoRPC).FullName);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestRPC.DummyContainerManager
(this), addr, conf, null, 1);
server.Start();
// Any unrelated protocol would do
ApplicationClientProtocol proxy = (ApplicationClientProtocol)rpc.GetProxy(typeof(
ApplicationClientProtocol), NetUtils.GetConnectAddress(server), conf);
try
{
proxy.GetNewApplication(Records.NewRecord <GetNewApplicationRequest>());
NUnit.Framework.Assert.Fail("Excepted RPC call to fail with unknown method.");
}
catch (YarnException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Matches("Unknown method getNewApplication called on.*"
+ "org.apache.hadoop.yarn.proto.ApplicationClientProtocol" + "\\$ApplicationClientProtocolService\\$BlockingInterface protocol."
));
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
}
internal virtual LocalizationProtocol GetProxy(IPEndPoint nmAddr)
{
YarnRPC rpc = YarnRPC.Create(conf);
return((LocalizationProtocol)rpc.GetProxy(typeof(LocalizationProtocol), nmAddr, conf
));
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
Configuration conf = GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
IPEndPoint masterServiceAddress = conf.GetSocketAddr(YarnConfiguration.RmBindHost
, YarnConfiguration.RmSchedulerAddress, YarnConfiguration.DefaultRmSchedulerAddress
, YarnConfiguration.DefaultRmSchedulerPort);
Configuration serverConf = conf;
// If the auth is not-simple, enforce it to be token-based.
serverConf = new Configuration(conf);
serverConf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, SaslRpcServer.AuthMethod
.Token.ToString());
this.server = rpc.GetServer(typeof(ApplicationMasterProtocol), this, masterServiceAddress
, serverConf, this.rmContext.GetAMRMTokenSecretManager(), serverConf.GetInt(YarnConfiguration
.RmSchedulerClientThreadCount, YarnConfiguration.DefaultRmSchedulerClientThreadCount
));
// Enable service authorization?
if (conf.GetBoolean(CommonConfigurationKeysPublic.HadoopSecurityAuthorization, false
))
{
InputStream inputStream = this.rmContext.GetConfigurationProvider().GetConfigurationInputStream
(conf, YarnConfiguration.HadoopPolicyConfigurationFile);
if (inputStream != null)
{
conf.AddResource(inputStream);
}
RefreshServiceAcls(conf, RMPolicyProvider.GetInstance());
}
this.server.Start();
this.bindAddress = conf.UpdateConnectAddr(YarnConfiguration.RmBindHost, YarnConfiguration
.RmSchedulerAddress, YarnConfiguration.DefaultRmSchedulerAddress, server.GetListenerAddress
());
base.ServiceStart();
}
public virtual void Setup()
{
conf = CreateConfiguration();
rpc = YarnRPC.Create(conf);
rmAddress = conf.GetSocketAddr(YarnConfiguration.RmAddress, YarnConfiguration.DefaultRmAddress
, YarnConfiguration.DefaultRmPort);
AccessControlList adminACL = new AccessControlList(string.Empty);
conf.Set(YarnConfiguration.YarnAdminAcl, adminACL.GetAclString());
resourceManager = new _MockRM_85(conf);
new _Thread_96(this).Start();
int waitCount = 0;
while (resourceManager.GetServiceState() == Service.STATE.Inited && waitCount++ <
60)
{
Log.Info("Waiting for RM to start...");
Sharpen.Thread.Sleep(1500);
}
if (resourceManager.GetServiceState() != Service.STATE.Started)
{
// RM could have failed.
throw new IOException("ResourceManager failed to start. Final state is " + resourceManager
.GetServiceState());
}
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
Configuration conf = GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
IPEndPoint address = new IPEndPoint(0);
server = rpc.GetServer(typeof(MRClientProtocol), protocolHandler, address, conf,
appContext.GetClientToAMTokenSecretManager(), conf.GetInt(MRJobConfig.MrAmJobClientThreadCount
, MRJobConfig.DefaultMrAmJobClientThreadCount), MRJobConfig.MrAmJobClientPortRange
);
// Enable service authorization?
if (conf.GetBoolean(CommonConfigurationKeysPublic.HadoopSecurityAuthorization, false
))
{
RefreshServiceAcls(conf, new MRAMPolicyProvider());
}
server.Start();
this.bindAddress = NetUtils.CreateSocketAddrForHost(appContext.GetNMHostname(), server
.GetListenerAddress().Port);
Log.Info("Instantiated MRClientService at " + this.bindAddress);
try
{
// Explicitly disabling SSL for map reduce task as we can't allow MR users
// to gain access to keystore file for opening SSL listener. We can trust
// RM/NM to issue SSL certificates but definitely not MR-AM as it is
// running in user-land.
webApp = WebApps.$for <AppContext>("mapreduce", appContext, "ws").WithHttpPolicy(conf
, HttpConfig.Policy.HttpOnly).Start(new AMWebApp());
}
catch (Exception e)
{
Log.Error("Webapps failed to start. Ignoring for now:", e);
}
base.ServiceStart();
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
Configuration conf = GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
IPEndPoint address = conf.GetSocketAddr(YarnConfiguration.TimelineServiceBindHost
, YarnConfiguration.TimelineServiceAddress, YarnConfiguration.DefaultTimelineServiceAddress
, YarnConfiguration.DefaultTimelineServicePort);
Preconditions.CheckArgument(conf.GetInt(YarnConfiguration.TimelineServiceHandlerThreadCount
, YarnConfiguration.DefaultTimelineServiceClientThreadCount) > 0, "%s property value should be greater than zero"
, YarnConfiguration.TimelineServiceHandlerThreadCount);
server = rpc.GetServer(typeof(ApplicationHistoryProtocol), this, address, conf, null
, conf.GetInt(YarnConfiguration.TimelineServiceHandlerThreadCount, YarnConfiguration
.DefaultTimelineServiceClientThreadCount));
// Enable service authorization?
if (conf.GetBoolean(CommonConfigurationKeysPublic.HadoopSecurityAuthorization, false
))
{
RefreshServiceAcls(conf, new TimelinePolicyProvider());
}
server.Start();
this.bindAddress = conf.UpdateConnectAddr(YarnConfiguration.TimelineServiceBindHost
, YarnConfiguration.TimelineServiceAddress, YarnConfiguration.DefaultTimelineServiceAddress
, server.GetListenerAddress());
Log.Info("Instantiated ApplicationHistoryClientService at " + this.bindAddress);
base.ServiceStart();
}
/// <exception cref="System.Exception"/>
protected internal virtual void StartServer()
{
Configuration conf = GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
this.server = (RPC.Server)rpc.GetServer(typeof(ResourceManagerAdministrationProtocol
), this, masterServiceBindAddress, conf, null, conf.GetInt(YarnConfiguration.RmAdminClientThreadCount
, YarnConfiguration.DefaultRmAdminClientThreadCount));
// Enable service authorization?
if (conf.GetBoolean(CommonConfigurationKeysPublic.HadoopSecurityAuthorization, false
))
{
RefreshServiceAcls(GetConfiguration(conf, YarnConfiguration.HadoopPolicyConfigurationFile
), RMPolicyProvider.GetInstance());
}
if (rmContext.IsHAEnabled())
{
RPC.SetProtocolEngine(conf, typeof(HAServiceProtocolPB), typeof(ProtobufRpcEngine
));
HAServiceProtocolServerSideTranslatorPB haServiceProtocolXlator = new HAServiceProtocolServerSideTranslatorPB
(this);
BlockingService haPbService = HAServiceProtocolProtos.HAServiceProtocolService.NewReflectiveBlockingService
(haServiceProtocolXlator);
server.AddProtocol(RPC.RpcKind.RpcProtocolBuffer, typeof(HAServiceProtocol), haPbService
);
}
this.server.Start();
conf.UpdateConnectAddr(YarnConfiguration.RmBindHost, YarnConfiguration.RmAdminAddress
, YarnConfiguration.DefaultRmAdminAddress, server.GetListenerAddress());
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
Configuration conf = GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
InitializeWebApp(conf);
IPEndPoint address = conf.GetSocketAddr(JHAdminConfig.MrHistoryBindHost, JHAdminConfig
.MrHistoryAddress, JHAdminConfig.DefaultMrHistoryAddress, JHAdminConfig.DefaultMrHistoryPort
);
server = rpc.GetServer(typeof(HSClientProtocol), protocolHandler, address, conf,
jhsDTSecretManager, conf.GetInt(JHAdminConfig.MrHistoryClientThreadCount, JHAdminConfig
.DefaultMrHistoryClientThreadCount));
// Enable service authorization?
if (conf.GetBoolean(CommonConfigurationKeysPublic.HadoopSecurityAuthorization, false
))
{
server.RefreshServiceAcl(conf, new ClientHSPolicyProvider());
}
server.Start();
this.bindAddress = conf.UpdateConnectAddr(JHAdminConfig.MrHistoryBindHost, JHAdminConfig
.MrHistoryAddress, JHAdminConfig.DefaultMrHistoryAddress, server.GetListenerAddress
());
Log.Info("Instantiated HistoryClientService at " + this.bindAddress);
base.ServiceStart();
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
base.ServiceStart();
// ResourceTrackerServer authenticates NodeManager via Kerberos if
// security is enabled, so no secretManager.
Configuration conf = GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
this.server = rpc.GetServer(typeof(ResourceTracker), this, resourceTrackerAddress
, conf, null, conf.GetInt(YarnConfiguration.RmResourceTrackerClientThreadCount,
YarnConfiguration.DefaultRmResourceTrackerClientThreadCount));
// Enable service authorization?
if (conf.GetBoolean(CommonConfigurationKeysPublic.HadoopSecurityAuthorization, false
))
{
InputStream inputStream = this.rmContext.GetConfigurationProvider().GetConfigurationInputStream
(conf, YarnConfiguration.HadoopPolicyConfigurationFile);
if (inputStream != null)
{
conf.AddResource(inputStream);
}
RefreshServiceAcls(conf, RMPolicyProvider.GetInstance());
}
this.server.Start();
conf.UpdateConnectAddr(YarnConfiguration.RmBindHost, YarnConfiguration.RmResourceTrackerAddress
, YarnConfiguration.DefaultRmResourceTrackerAddress, server.GetListenerAddress()
);
}
public virtual void Start()
{
Configuration conf = new Configuration();
YarnRPC rpc = YarnRPC.Create(conf);
server = rpc.GetServer(typeof(LocalizationProtocol), this, locAddr, conf, null, 1
);
server.Start();
}
private HSClientProtocol InstantiateHistoryProxy()
{
string serviceAddr = mrCluster.GetConfig().Get(JHAdminConfig.MrHistoryAddress);
YarnRPC rpc = YarnRPC.Create(conf);
HSClientProtocol historyClient = (HSClientProtocol)rpc.GetProxy(typeof(HSClientProtocol
), NetUtils.CreateSocketAddr(serviceAddr), mrCluster.GetConfig());
return(historyClient);
}
/// <summary>This tests whether a containerId is serialized/deserialized with epoch.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private void TestContainerTokenWithEpoch(Configuration conf)
{
Log.Info("Running test for serializing/deserializing containerIds");
NMTokenSecretManagerInRM nmTokenSecretManagerInRM = yarnCluster.GetResourceManager
().GetRMContext().GetNMTokenSecretManager();
ApplicationId appId = ApplicationId.NewInstance(1, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 0);
ContainerId cId = ContainerId.NewContainerId(appAttemptId, (5L << 40) | 3L);
NodeManager nm = yarnCluster.GetNodeManager(0);
NMTokenSecretManagerInNM nmTokenSecretManagerInNM = nm.GetNMContext().GetNMTokenSecretManager
();
string user = "******";
WaitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm);
NodeId nodeId = nm.GetNMContext().GetNodeId();
// Both id should be equal.
NUnit.Framework.Assert.AreEqual(nmTokenSecretManagerInNM.GetCurrentKey().GetKeyId
(), nmTokenSecretManagerInRM.GetCurrentKey().GetKeyId());
// Creating a normal Container Token
RMContainerTokenSecretManager containerTokenSecretManager = yarnCluster.GetResourceManager
().GetRMContext().GetContainerTokenSecretManager();
Resource r = Resource.NewInstance(1230, 2);
Org.Apache.Hadoop.Yarn.Api.Records.Token containerToken = containerTokenSecretManager
.CreateContainerToken(cId, nodeId, user, r, Priority.NewInstance(0), 0);
ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
);
byte[] tokenIdentifierContent = ((byte[])containerToken.GetIdentifier().Array());
DataInputBuffer dib = new DataInputBuffer();
dib.Reset(tokenIdentifierContent, tokenIdentifierContent.Length);
containerTokenIdentifier.ReadFields(dib);
NUnit.Framework.Assert.AreEqual(cId, containerTokenIdentifier.GetContainerID());
NUnit.Framework.Assert.AreEqual(cId.ToString(), containerTokenIdentifier.GetContainerID
().ToString());
Org.Apache.Hadoop.Yarn.Api.Records.Token nmToken = nmTokenSecretManagerInRM.CreateNMToken
(appAttemptId, nodeId, user);
YarnRPC rpc = YarnRPC.Create(conf);
TestStartContainer(rpc, appAttemptId, nodeId, containerToken, nmToken, false);
IList <ContainerId> containerIds = new List <ContainerId>();
containerIds.AddItem(cId);
ContainerManagementProtocol proxy = GetContainerManagementProtocolProxy(rpc, nmToken
, nodeId, user);
GetContainerStatusesResponse res = proxy.GetContainerStatuses(GetContainerStatusesRequest
.NewInstance(containerIds));
NUnit.Framework.Assert.IsNotNull(res.GetContainerStatuses()[0]);
NUnit.Framework.Assert.AreEqual(cId, res.GetContainerStatuses()[0].GetContainerId
());
NUnit.Framework.Assert.AreEqual(cId.ToString(), res.GetContainerStatuses()[0].GetContainerId
().ToString());
}
private SCMUploaderProtocol CreateSCMClient(Configuration conf)
{
YarnRPC rpc = YarnRPC.Create(conf);
IPEndPoint scmAddress = conf.GetSocketAddr(YarnConfiguration.ScmUploaderServerAddress
, YarnConfiguration.DefaultScmUploaderServerAddress, YarnConfiguration.DefaultScmUploaderServerPort
);
return((SCMUploaderProtocol)rpc.GetProxy(typeof(SCMUploaderProtocol), scmAddress,
conf));
}
public ContainerManagementProtocol Run()
{
Configuration conf = new Configuration();
YarnRPC rpc = YarnRPC.Create(conf);
IPEndPoint containerManagerBindAddress = NetUtils.CreateSocketAddrForHost("127.0.0.1"
, 12345);
return((ContainerManagementProtocol)rpc.GetProxy(typeof(ContainerManagementProtocol
), containerManagerBindAddress, conf));
}
public virtual void TestValidateResourceBlacklistRequest()
{
TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
();
TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(new
YarnConfiguration(), containerManager);
rm.Start();
MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>(2);
acls[ApplicationAccessType.ViewApp] = "*";
RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls);
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();
WaitForLaunchedState(attempt);
// Create a client to the RM.
Configuration conf = rm.GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
.ToString());
Credentials credentials = containerManager.GetContainerCredentials();
IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
.SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
currentUser.AddToken(amRMToken);
ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_626(rpc
, rmBindAddress, conf));
RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<RegisterApplicationMasterRequest>();
client.RegisterApplicationMaster(request);
ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance(
Sharpen.Collections.SingletonList(ResourceRequest.Any), null);
AllocateRequest allocateRequest = AllocateRequest.NewInstance(0, 0.0f, null, null
, blacklistRequest);
bool error = false;
try
{
client.Allocate(allocateRequest);
}
catch (InvalidResourceBlacklistRequestException)
{
error = true;
}
rm.Stop();
NUnit.Framework.Assert.IsTrue("Didn't not catch InvalidResourceBlacklistRequestException"
, error);
}
/// <exception cref="System.IO.IOException"/>
protected override GetUserMappingsProtocol GetUgmProtocol()
{
Configuration conf = GetConf();
IPEndPoint addr = conf.GetSocketAddr(YarnConfiguration.RmAdminAddress, YarnConfiguration
.DefaultRmAdminAddress, YarnConfiguration.DefaultRmAdminPort);
YarnRPC rpc = YarnRPC.Create(conf);
ResourceManagerAdministrationProtocol adminProtocol = (ResourceManagerAdministrationProtocol
)rpc.GetProxy(typeof(ResourceManagerAdministrationProtocol), addr, GetConf());
return(adminProtocol);
}
/// <exception cref="System.IO.IOException"/>
internal virtual MRClientProtocol InstantiateAMProxy(IPEndPoint serviceAddr)
{
Log.Trace("Connecting to ApplicationMaster at: " + serviceAddr);
YarnRPC rpc = YarnRPC.Create(conf);
MRClientProtocol proxy = (MRClientProtocol)rpc.GetProxy(typeof(MRClientProtocol),
serviceAddr, conf);
usingAMProxy.Set(true);
Log.Trace("Connected to ApplicationMaster at: " + serviceAddr);
return(proxy);
}
/// <exception cref="System.Exception"/>
private void TestRPCTimeout(string rpcClass)
{
Configuration conf = new Configuration();
// set timeout low for the test
conf.SetInt("yarn.rpc.nm-command-timeout", 3000);
conf.Set(YarnConfiguration.IpcRpcImpl, rpcClass);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestContainerLaunchRPC.DummyContainerManager
(this), addr, conf, null, 1);
server.Start();
try
{
ContainerManagementProtocol proxy = (ContainerManagementProtocol)rpc.GetProxy(typeof(
ContainerManagementProtocol), server.GetListenerAddress(), conf);
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
ApplicationId applicationId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 0);
ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 100);
NodeId nodeId = NodeId.NewInstance("localhost", 1234);
Resource resource = Resource.NewInstance(1234, 2);
ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
containerId, "localhost", "user", resource, Runtime.CurrentTimeMillis() + 10000,
42, 42, Priority.NewInstance(0), 0);
Token containerToken = TestRPC.NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString
("password"), containerTokenIdentifier);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
try
{
proxy.StartContainers(allRequests);
}
catch (Exception e)
{
Log.Info(StringUtils.StringifyException(e));
NUnit.Framework.Assert.AreEqual("Error, exception is not: " + typeof(SocketTimeoutException
).FullName, typeof(SocketTimeoutException).FullName, e.GetType().FullName);
return;
}
}
finally
{
server.Stop();
}
NUnit.Framework.Assert.Fail("timeout exception should have occurred!");
}
public virtual void TestAuthorizedAccess()
{
TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
();
rm = new TestAMAuthorization.MockRMWithAMS(conf, containerManager);
rm.Start();
MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>(2);
acls[ApplicationAccessType.ViewApp] = "*";
RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls);
nm1.NodeHeartbeat(true);
int waitCount = 0;
while (containerManager.containerTokens == null && waitCount++ < 20)
{
Log.Info("Waiting for AM Launch to happen..");
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();
WaitForLaunchedState(attempt);
// Create a client to the RM.
Configuration conf = rm.GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
.ToString());
Credentials credentials = containerManager.GetContainerCredentials();
IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
.SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
currentUser.AddToken(amRMToken);
ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_206(this
, rpc, conf));
RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<RegisterApplicationMasterRequest>();
RegisterApplicationMasterResponse response = client.RegisterApplicationMaster(request
);
NUnit.Framework.Assert.IsNotNull(response.GetClientToAMTokenMasterKey());
if (UserGroupInformation.IsSecurityEnabled())
{
NUnit.Framework.Assert.IsTrue(((byte[])response.GetClientToAMTokenMasterKey().Array
()).Length > 0);
}
NUnit.Framework.Assert.AreEqual("Register response has bad ACLs", "*", response.GetApplicationACLs
()[ApplicationAccessType.ViewApp]);
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual MRClientProtocol InstantiateHistoryProxy(Configuration
conf, IPEndPoint hsAddress)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Connecting to MRHistoryServer at: " + hsAddress);
}
YarnRPC rpc = YarnRPC.Create(conf);
UserGroupInformation currentUser = UserGroupInformation.GetCurrentUser();
return(currentUser.DoAs(new _PrivilegedAction_113(rpc, hsAddress, conf)));
}
private MRClientProtocol GetMRClientProtocol(Token token, IPEndPoint hsAddress, string
user, Configuration conf)
{
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(user);
ugi.AddToken(ConverterUtils.ConvertFromYarn(token, hsAddress));
YarnRPC rpc = YarnRPC.Create(conf);
MRClientProtocol hsWithDT = ugi.DoAs(new _PrivilegedAction_288(rpc, hsAddress, conf
));
return(hsWithDT);
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
// All the clients to appsManager are supposed to be authenticated via
// Kerberos if security is enabled, so no secretManager.
YarnRPC rpc = YarnRPC.Create(this.GetConfig());
Configuration clientServerConf = new Configuration(this.GetConfig());
this.server = rpc.GetServer(typeof(ApplicationClientProtocol), this, this.clientBindAddress
, clientServerConf, null, 1);
this.server.Start();
base.ServiceStart();
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual SCMAdminProtocol CreateSCMAdminProtocol()
{
// Get the current configuration
YarnConfiguration conf = new YarnConfiguration(GetConf());
// Create the admin client
IPEndPoint addr = conf.GetSocketAddr(YarnConfiguration.ScmAdminAddress, YarnConfiguration
.DefaultScmAdminAddress, YarnConfiguration.DefaultScmAdminPort);
YarnRPC rpc = YarnRPC.Create(conf);
SCMAdminProtocol scmAdminProtocol = (SCMAdminProtocol)rpc.GetProxy(typeof(SCMAdminProtocol
), addr, conf);
return(scmAdminProtocol);
}
private ApplicationClientProtocol GetClientRMProtocolWithDT(Org.Apache.Hadoop.Yarn.Api.Records.Token
token, IPEndPoint rmAddress, string user, Configuration conf)
{
// Maybe consider converting to Hadoop token, serialize de-serialize etc
// before trying to renew the token.
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(user);
ugi.AddToken(ConverterUtils.ConvertFromYarn(token, rmAddress));
YarnRPC rpc = YarnRPC.Create(conf);
ApplicationClientProtocol clientRMWithDT = ugi.DoAs(new _PrivilegedAction_464(rpc
, rmAddress, conf));
return(clientRMWithDT);
}
/// <exception cref="System.Exception"/>
public virtual void TestSlowNM()
{
conf = new Configuration();
int maxAttempts = 1;
conf.SetInt(MRJobConfig.MapMaxAttempts, maxAttempts);
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
// set timeout low for the test
conf.SetInt("yarn.rpc.nm-command-timeout", 3000);
conf.Set(YarnConfiguration.IpcRpcImpl, typeof(HadoopYarnProtoRPC).FullName);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
NMTokenSecretManagerInNM tokenSecretManager = new NMTokenSecretManagerInNM();
MasterKey masterKey = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord<MasterKey>();
masterKey.SetBytes(ByteBuffer.Wrap(Sharpen.Runtime.GetBytesForString("key")));
tokenSecretManager.SetMasterKey(masterKey);
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "token");
server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestContainerLauncher.DummyContainerManager
(this), addr, conf, tokenSecretManager, 1);
server.Start();
MRApp app = new TestContainerLauncher.MRAppWithSlowNM(this, tokenSecretManager);
try
{
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
IDictionary<TaskId, Task> tasks = job.GetTasks();
NUnit.Framework.Assert.AreEqual("Num tasks is not correct", 1, tasks.Count);
Task task = tasks.Values.GetEnumerator().Next();
app.WaitForState(task, TaskState.Scheduled);
IDictionary<TaskAttemptId, TaskAttempt> attempts = tasks.Values.GetEnumerator().Next
().GetAttempts();
NUnit.Framework.Assert.AreEqual("Num attempts is not correct", maxAttempts, attempts
.Count);
TaskAttempt attempt = attempts.Values.GetEnumerator().Next();
app.WaitForInternalState((TaskAttemptImpl)attempt, TaskAttemptStateInternal.Assigned
);
app.WaitForState(job, JobState.Failed);
string diagnostics = attempt.GetDiagnostics().ToString();
Log.Info("attempt.getDiagnostics: " + diagnostics);
NUnit.Framework.Assert.IsTrue(diagnostics.Contains("Container launch failed for "
+ "container_0_0000_01_000000 : "));
NUnit.Framework.Assert.IsTrue(diagnostics.Contains("java.net.SocketTimeoutException: 3000 millis timeout while waiting for channel"
));
}
finally
{
server.Stop();
app.Stop();
}
}
GetCMProxy(string containerMgrBindAddr, ContainerId containerId)
{
IPEndPoint addr = NetUtils.GetConnectAddress(this._enclosing._enclosing.server);
string containerManagerBindAddr = addr.GetHostName() + ":" + addr.Port;
Token token = this._enclosing.tokenSecretManager.CreateNMToken(containerId.GetApplicationAttemptId
(), NodeId.NewInstance(addr.GetHostName(), addr.Port), "user");
ContainerManagementProtocolProxy cmProxy = new ContainerManagementProtocolProxy(this
._enclosing._enclosing.conf);
ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData proxy = new
ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData(this, YarnRPC
.Create(this._enclosing._enclosing.conf), containerManagerBindAddr, containerId,
token);
return proxy;
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
Configuration conf = GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
this.server = rpc.GetServer(typeof(SCMAdminProtocol), this, clientBindAddress, conf
, null, conf.GetInt(YarnConfiguration.ScmAdminClientThreadCount, YarnConfiguration
.DefaultScmAdminClientThreadCount));
// Secret manager null for now (security not supported)
// TODO: Enable service authorization (see YARN-2774)
this.server.Start();
clientBindAddress = conf.UpdateConnectAddr(YarnConfiguration.ScmAdminAddress, server
.GetListenerAddress());
base.ServiceStart();
}