/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
Configuration conf = GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
IPEndPoint address = new IPEndPoint(0);
server = rpc.GetServer(typeof(MRClientProtocol), protocolHandler, address, conf,
appContext.GetClientToAMTokenSecretManager(), conf.GetInt(MRJobConfig.MrAmJobClientThreadCount
, MRJobConfig.DefaultMrAmJobClientThreadCount), MRJobConfig.MrAmJobClientPortRange
);
// Enable service authorization?
if (conf.GetBoolean(CommonConfigurationKeysPublic.HadoopSecurityAuthorization, false
))
{
RefreshServiceAcls(conf, new MRAMPolicyProvider());
}
server.Start();
this.bindAddress = NetUtils.CreateSocketAddrForHost(appContext.GetNMHostname(), server
.GetListenerAddress().Port);
Log.Info("Instantiated MRClientService at " + this.bindAddress);
try
{
// Explicitly disabling SSL for map reduce task as we can't allow MR users
// to gain access to keystore file for opening SSL listener. We can trust
// RM/NM to issue SSL certificates but definitely not MR-AM as it is
// running in user-land.
webApp = WebApps.$for <AppContext>("mapreduce", appContext, "ws").WithHttpPolicy(conf
, HttpConfig.Policy.HttpOnly).Start(new AMWebApp());
}
catch (Exception e)
{
Log.Error("Webapps failed to start. Ignoring for now:", e);
}
base.ServiceStart();
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
Configuration conf = GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
InitializeWebApp(conf);
IPEndPoint address = conf.GetSocketAddr(JHAdminConfig.MrHistoryBindHost, JHAdminConfig
.MrHistoryAddress, JHAdminConfig.DefaultMrHistoryAddress, JHAdminConfig.DefaultMrHistoryPort
);
server = rpc.GetServer(typeof(HSClientProtocol), protocolHandler, address, conf,
jhsDTSecretManager, conf.GetInt(JHAdminConfig.MrHistoryClientThreadCount, JHAdminConfig
.DefaultMrHistoryClientThreadCount));
// Enable service authorization?
if (conf.GetBoolean(CommonConfigurationKeysPublic.HadoopSecurityAuthorization, false
))
{
server.RefreshServiceAcl(conf, new ClientHSPolicyProvider());
}
server.Start();
this.bindAddress = conf.UpdateConnectAddr(JHAdminConfig.MrHistoryBindHost, JHAdminConfig
.MrHistoryAddress, JHAdminConfig.DefaultMrHistoryAddress, server.GetListenerAddress
());
Log.Info("Instantiated HistoryClientService at " + this.bindAddress);
base.ServiceStart();
}
public virtual void TestUnknownCall()
{
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.IpcRpcImpl, typeof(HadoopYarnProtoRPC).FullName);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestRPC.DummyContainerManager
(this), addr, conf, null, 1);
server.Start();
// Any unrelated protocol would do
ApplicationClientProtocol proxy = (ApplicationClientProtocol)rpc.GetProxy(typeof(
ApplicationClientProtocol), NetUtils.GetConnectAddress(server), conf);
try
{
proxy.GetNewApplication(Records.NewRecord <GetNewApplicationRequest>());
NUnit.Framework.Assert.Fail("Excepted RPC call to fail with unknown method.");
}
catch (YarnException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Matches("Unknown method getNewApplication called on.*"
+ "org.apache.hadoop.yarn.proto.ApplicationClientProtocol" + "\\$ApplicationClientProtocolService\\$BlockingInterface protocol."
));
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
Configuration conf = GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
IPEndPoint address = conf.GetSocketAddr(YarnConfiguration.TimelineServiceBindHost
, YarnConfiguration.TimelineServiceAddress, YarnConfiguration.DefaultTimelineServiceAddress
, YarnConfiguration.DefaultTimelineServicePort);
Preconditions.CheckArgument(conf.GetInt(YarnConfiguration.TimelineServiceHandlerThreadCount
, YarnConfiguration.DefaultTimelineServiceClientThreadCount) > 0, "%s property value should be greater than zero"
, YarnConfiguration.TimelineServiceHandlerThreadCount);
server = rpc.GetServer(typeof(ApplicationHistoryProtocol), this, address, conf, null
, conf.GetInt(YarnConfiguration.TimelineServiceHandlerThreadCount, YarnConfiguration
.DefaultTimelineServiceClientThreadCount));
// Enable service authorization?
if (conf.GetBoolean(CommonConfigurationKeysPublic.HadoopSecurityAuthorization, false
))
{
RefreshServiceAcls(conf, new TimelinePolicyProvider());
}
server.Start();
this.bindAddress = conf.UpdateConnectAddr(YarnConfiguration.TimelineServiceBindHost
, YarnConfiguration.TimelineServiceAddress, YarnConfiguration.DefaultTimelineServiceAddress
, server.GetListenerAddress());
Log.Info("Instantiated ApplicationHistoryClientService at " + this.bindAddress);
base.ServiceStart();
}
/// <exception cref="System.Exception"/>
protected internal virtual void StartServer()
{
Configuration conf = GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
this.server = (RPC.Server)rpc.GetServer(typeof(ResourceManagerAdministrationProtocol
), this, masterServiceBindAddress, conf, null, conf.GetInt(YarnConfiguration.RmAdminClientThreadCount
, YarnConfiguration.DefaultRmAdminClientThreadCount));
// Enable service authorization?
if (conf.GetBoolean(CommonConfigurationKeysPublic.HadoopSecurityAuthorization, false
))
{
RefreshServiceAcls(GetConfiguration(conf, YarnConfiguration.HadoopPolicyConfigurationFile
), RMPolicyProvider.GetInstance());
}
if (rmContext.IsHAEnabled())
{
RPC.SetProtocolEngine(conf, typeof(HAServiceProtocolPB), typeof(ProtobufRpcEngine
));
HAServiceProtocolServerSideTranslatorPB haServiceProtocolXlator = new HAServiceProtocolServerSideTranslatorPB
(this);
BlockingService haPbService = HAServiceProtocolProtos.HAServiceProtocolService.NewReflectiveBlockingService
(haServiceProtocolXlator);
server.AddProtocol(RPC.RpcKind.RpcProtocolBuffer, typeof(HAServiceProtocol), haPbService
);
}
this.server.Start();
conf.UpdateConnectAddr(YarnConfiguration.RmBindHost, YarnConfiguration.RmAdminAddress
, YarnConfiguration.DefaultRmAdminAddress, server.GetListenerAddress());
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
base.ServiceStart();
// ResourceTrackerServer authenticates NodeManager via Kerberos if
// security is enabled, so no secretManager.
Configuration conf = GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
this.server = rpc.GetServer(typeof(ResourceTracker), this, resourceTrackerAddress
, conf, null, conf.GetInt(YarnConfiguration.RmResourceTrackerClientThreadCount,
YarnConfiguration.DefaultRmResourceTrackerClientThreadCount));
// Enable service authorization?
if (conf.GetBoolean(CommonConfigurationKeysPublic.HadoopSecurityAuthorization, false
))
{
InputStream inputStream = this.rmContext.GetConfigurationProvider().GetConfigurationInputStream
(conf, YarnConfiguration.HadoopPolicyConfigurationFile);
if (inputStream != null)
{
conf.AddResource(inputStream);
}
RefreshServiceAcls(conf, RMPolicyProvider.GetInstance());
}
this.server.Start();
conf.UpdateConnectAddr(YarnConfiguration.RmBindHost, YarnConfiguration.RmResourceTrackerAddress
, YarnConfiguration.DefaultRmResourceTrackerAddress, server.GetListenerAddress()
);
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
Configuration conf = GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
IPEndPoint masterServiceAddress = conf.GetSocketAddr(YarnConfiguration.RmBindHost
, YarnConfiguration.RmSchedulerAddress, YarnConfiguration.DefaultRmSchedulerAddress
, YarnConfiguration.DefaultRmSchedulerPort);
Configuration serverConf = conf;
// If the auth is not-simple, enforce it to be token-based.
serverConf = new Configuration(conf);
serverConf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, SaslRpcServer.AuthMethod
.Token.ToString());
this.server = rpc.GetServer(typeof(ApplicationMasterProtocol), this, masterServiceAddress
, serverConf, this.rmContext.GetAMRMTokenSecretManager(), serverConf.GetInt(YarnConfiguration
.RmSchedulerClientThreadCount, YarnConfiguration.DefaultRmSchedulerClientThreadCount
));
// Enable service authorization?
if (conf.GetBoolean(CommonConfigurationKeysPublic.HadoopSecurityAuthorization, false
))
{
InputStream inputStream = this.rmContext.GetConfigurationProvider().GetConfigurationInputStream
(conf, YarnConfiguration.HadoopPolicyConfigurationFile);
if (inputStream != null)
{
conf.AddResource(inputStream);
}
RefreshServiceAcls(conf, RMPolicyProvider.GetInstance());
}
this.server.Start();
this.bindAddress = conf.UpdateConnectAddr(YarnConfiguration.RmBindHost, YarnConfiguration
.RmSchedulerAddress, YarnConfiguration.DefaultRmSchedulerAddress, server.GetListenerAddress
());
base.ServiceStart();
}
public virtual void Start()
{
Configuration conf = new Configuration();
YarnRPC rpc = YarnRPC.Create(conf);
server = rpc.GetServer(typeof(LocalizationProtocol), this, locAddr, conf, null, 1
);
server.Start();
}
/// <exception cref="System.Exception"/>
private void TestRPCTimeout(string rpcClass)
{
Configuration conf = new Configuration();
// set timeout low for the test
conf.SetInt("yarn.rpc.nm-command-timeout", 3000);
conf.Set(YarnConfiguration.IpcRpcImpl, rpcClass);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestContainerLaunchRPC.DummyContainerManager
(this), addr, conf, null, 1);
server.Start();
try
{
ContainerManagementProtocol proxy = (ContainerManagementProtocol)rpc.GetProxy(typeof(
ContainerManagementProtocol), server.GetListenerAddress(), conf);
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
ApplicationId applicationId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 0);
ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 100);
NodeId nodeId = NodeId.NewInstance("localhost", 1234);
Resource resource = Resource.NewInstance(1234, 2);
ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
containerId, "localhost", "user", resource, Runtime.CurrentTimeMillis() + 10000,
42, 42, Priority.NewInstance(0), 0);
Token containerToken = TestRPC.NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString
("password"), containerTokenIdentifier);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
try
{
proxy.StartContainers(allRequests);
}
catch (Exception e)
{
Log.Info(StringUtils.StringifyException(e));
NUnit.Framework.Assert.AreEqual("Error, exception is not: " + typeof(SocketTimeoutException
).FullName, typeof(SocketTimeoutException).FullName, e.GetType().FullName);
return;
}
}
finally
{
server.Stop();
}
NUnit.Framework.Assert.Fail("timeout exception should have occurred!");
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
// All the clients to appsManager are supposed to be authenticated via
// Kerberos if security is enabled, so no secretManager.
YarnRPC rpc = YarnRPC.Create(this.GetConfig());
Configuration clientServerConf = new Configuration(this.GetConfig());
this.server = rpc.GetServer(typeof(ApplicationClientProtocol), this, this.clientBindAddress
, clientServerConf, null, 1);
this.server.Start();
base.ServiceStart();
}
/// <exception cref="System.Exception"/>
public virtual void TestSlowNM()
{
conf = new Configuration();
int maxAttempts = 1;
conf.SetInt(MRJobConfig.MapMaxAttempts, maxAttempts);
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
// set timeout low for the test
conf.SetInt("yarn.rpc.nm-command-timeout", 3000);
conf.Set(YarnConfiguration.IpcRpcImpl, typeof(HadoopYarnProtoRPC).FullName);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
NMTokenSecretManagerInNM tokenSecretManager = new NMTokenSecretManagerInNM();
MasterKey masterKey = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord<MasterKey>();
masterKey.SetBytes(ByteBuffer.Wrap(Sharpen.Runtime.GetBytesForString("key")));
tokenSecretManager.SetMasterKey(masterKey);
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "token");
server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestContainerLauncher.DummyContainerManager
(this), addr, conf, tokenSecretManager, 1);
server.Start();
MRApp app = new TestContainerLauncher.MRAppWithSlowNM(this, tokenSecretManager);
try
{
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
IDictionary<TaskId, Task> tasks = job.GetTasks();
NUnit.Framework.Assert.AreEqual("Num tasks is not correct", 1, tasks.Count);
Task task = tasks.Values.GetEnumerator().Next();
app.WaitForState(task, TaskState.Scheduled);
IDictionary<TaskAttemptId, TaskAttempt> attempts = tasks.Values.GetEnumerator().Next
().GetAttempts();
NUnit.Framework.Assert.AreEqual("Num attempts is not correct", maxAttempts, attempts
.Count);
TaskAttempt attempt = attempts.Values.GetEnumerator().Next();
app.WaitForInternalState((TaskAttemptImpl)attempt, TaskAttemptStateInternal.Assigned
);
app.WaitForState(job, JobState.Failed);
string diagnostics = attempt.GetDiagnostics().ToString();
Log.Info("attempt.getDiagnostics: " + diagnostics);
NUnit.Framework.Assert.IsTrue(diagnostics.Contains("Container launch failed for "
+ "container_0_0000_01_000000 : "));
NUnit.Framework.Assert.IsTrue(diagnostics.Contains("java.net.SocketTimeoutException: 3000 millis timeout while waiting for channel"
));
}
finally
{
server.Stop();
app.Stop();
}
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
Configuration conf = GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
this.server = rpc.GetServer(typeof(SCMAdminProtocol), this, clientBindAddress, conf
, null, conf.GetInt(YarnConfiguration.ScmAdminClientThreadCount, YarnConfiguration
.DefaultScmAdminClientThreadCount));
// Secret manager null for now (security not supported)
// TODO: Enable service authorization (see YARN-2774)
this.server.Start();
clientBindAddress = conf.UpdateConnectAddr(YarnConfiguration.ScmAdminAddress, server
.GetListenerAddress());
base.ServiceStart();
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
Configuration conf = GetConfig();
this.metrics = SharedCacheUploaderMetrics.GetInstance();
YarnRPC rpc = YarnRPC.Create(conf);
this.server = rpc.GetServer(typeof(SCMUploaderProtocol), this, bindAddress, conf,
null, conf.GetInt(YarnConfiguration.ScmUploaderServerThreadCount, YarnConfiguration
.DefaultScmUploaderServerThreadCount));
// Secret manager null for now (security not supported)
// TODO (YARN-2774): Enable service authorization
this.server.Start();
bindAddress = conf.UpdateConnectAddr(YarnConfiguration.ScmUploaderServerAddress,
server.GetListenerAddress());
base.ServiceStart();
}
public virtual void Start(Configuration conf)
{
YarnRPC rpc = YarnRPC.Create(conf);
//TODO : use fixed port ??
IPEndPoint address = NetUtils.CreateSocketAddr(this.hostAddress);
IPAddress hostNameResolved = null;
try
{
address.Address;
hostNameResolved = Sharpen.Runtime.GetLocalHost();
}
catch (UnknownHostException e)
{
throw new YarnRuntimeException(e);
}
this.server = rpc.GetServer(this.protocol, this, address, conf, null, 1);
this.server.Start();
this.bindAddress = NetUtils.GetConnectAddress(this.server);
base.Start();
this._enclosing.amRunning = true;
}
/// <exception cref="System.Exception"/>
private void Test(string rpcClass)
{
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.IpcRpcImpl, rpcClass);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestRPC.DummyContainerManager
(this), addr, conf, null, 1);
server.Start();
RPC.SetProtocolEngine(conf, typeof(ContainerManagementProtocolPB), typeof(ProtobufRpcEngine
));
ContainerManagementProtocol proxy = (ContainerManagementProtocol)rpc.GetProxy(typeof(
ContainerManagementProtocol), NetUtils.GetConnectAddress(server), conf);
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
ApplicationId applicationId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 0);
ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 100);
NodeId nodeId = NodeId.NewInstance("localhost", 1234);
Resource resource = Resource.NewInstance(1234, 2);
ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
containerId, "localhost", "user", resource, Runtime.CurrentTimeMillis() + 10000,
42, 42, Priority.NewInstance(0), 0);
Token containerToken = NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString
("password"), containerTokenIdentifier);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
proxy.StartContainers(allRequests);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(containerId);
GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance(
containerIds);
GetContainerStatusesResponse response = proxy.GetContainerStatuses(gcsRequest);
IList <ContainerStatus> statuses = response.GetContainerStatuses();
//test remote exception
bool exception = false;
try
{
StopContainersRequest stopRequest = recordFactory.NewRecordInstance <StopContainersRequest
>();
stopRequest.SetContainerIds(containerIds);
proxy.StopContainers(stopRequest);
}
catch (YarnException e)
{
exception = true;
NUnit.Framework.Assert.IsTrue(e.Message.Contains(ExceptionMsg));
NUnit.Framework.Assert.IsTrue(e.Message.Contains(ExceptionCause));
System.Console.Out.WriteLine("Test Exception is " + e.Message);
}
catch (Exception ex)
{
Sharpen.Runtime.PrintStackTrace(ex);
}
NUnit.Framework.Assert.IsTrue(exception);
server.Stop();
NUnit.Framework.Assert.IsNotNull(statuses[0]);
NUnit.Framework.Assert.AreEqual(ContainerState.Running, statuses[0].GetState());
}