protected internal virtual void StartRpcServer() { Configuration conf = GetConfig(); try { server = new RPC.Builder(conf).SetProtocol(typeof(TaskUmbilicalProtocol)).SetInstance (this).SetBindAddress("0.0.0.0").SetPort(0).SetNumHandlers(conf.GetInt(MRJobConfig .MrAmTaskListenerThreadCount, MRJobConfig.DefaultMrAmTaskListenerThreadCount)).SetVerbose (false).SetSecretManager(jobTokenSecretManager).Build(); // Enable service authorization? if (conf.GetBoolean(CommonConfigurationKeysPublic.HadoopSecurityAuthorization, false )) { RefreshServiceAcls(conf, new MRAMPolicyProvider()); } server.Start(); this.address = NetUtils.CreateSocketAddrForHost(context.GetNMHostname(), server.GetListenerAddress ().Port); } catch (IOException e) { throw new YarnRuntimeException(e); } }
/// <exception cref="System.Exception"/> protected override void ServiceStart() { Configuration conf = GetConfig(); YarnRPC rpc = YarnRPC.Create(conf); IPEndPoint address = new IPEndPoint(0); server = rpc.GetServer(typeof(MRClientProtocol), protocolHandler, address, conf, appContext.GetClientToAMTokenSecretManager(), conf.GetInt(MRJobConfig.MrAmJobClientThreadCount , MRJobConfig.DefaultMrAmJobClientThreadCount), MRJobConfig.MrAmJobClientPortRange ); // Enable service authorization? if (conf.GetBoolean(CommonConfigurationKeysPublic.HadoopSecurityAuthorization, false )) { RefreshServiceAcls(conf, new MRAMPolicyProvider()); } server.Start(); this.bindAddress = NetUtils.CreateSocketAddrForHost(appContext.GetNMHostname(), server .GetListenerAddress().Port); Log.Info("Instantiated MRClientService at " + this.bindAddress); try { // Explicitly disabling SSL for map reduce task as we can't allow MR users // to gain access to keystore file for opening SSL listener. We can trust // RM/NM to issue SSL certificates but definitely not MR-AM as it is // running in user-land. webApp = WebApps.$for <AppContext>("mapreduce", appContext, "ws").WithHttpPolicy(conf , HttpConfig.Policy.HttpOnly).Start(new AMWebApp()); } catch (Exception e) { Log.Error("Webapps failed to start. Ignoring for now:", e); } base.ServiceStart(); }
public virtual void TestGoodHostsAndPorts() { IPEndPoint compare = NetUtils.CreateSocketAddrForHost("localhost", 123); RunGoodCases(compare, "localhost", 123); RunGoodCases(compare, "localhost:", 123); RunGoodCases(compare, "localhost:123", 456); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public static void StartContainer(NodeManager nm, ContainerId cId, FileContext localFS , FilePath scriptFileDir, FilePath processStartFile) { FilePath scriptFile = CreateUnhaltingScriptFile(cId, scriptFileDir, processStartFile ); ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); NodeId nodeId = BuilderUtils.NewNodeId(Sharpen.Extensions.GetAddressByName("localhost" ).ToString(), 12345); URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new Path(scriptFile.GetAbsolutePath()))); LocalResource localResource = recordFactory.NewRecordInstance <LocalResource>(); localResource.SetResource(localResourceUri); localResource.SetSize(-1); localResource.SetVisibility(LocalResourceVisibility.Application); localResource.SetType(LocalResourceType.File); localResource.SetTimestamp(scriptFile.LastModified()); string destinationFile = "dest_file"; IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); localResources[destinationFile] = localResource; containerLaunchContext.SetLocalResources(localResources); IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile)); containerLaunchContext.SetCommands(commands); IPEndPoint containerManagerBindAddress = NetUtils.CreateSocketAddrForHost("127.0.0.1" , 12345); UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.ToString ()); Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> nmToken = ConverterUtils .ConvertFromYarn(nm.GetNMContext().GetNMTokenSecretManager().CreateNMToken(cId.GetApplicationAttemptId (), nodeId, user), containerManagerBindAddress); currentUser.AddToken(nmToken); ContainerManagementProtocol containerManager = currentUser.DoAs(new _PrivilegedAction_229 ()); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , TestContainerManager.CreateContainerToken(cId, 0, nodeId, user, nm.GetNMContext ().GetContainerTokenSecretManager())); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); containerManager.StartContainers(allRequests); IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(cId); GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds ); ContainerStatus containerStatus = containerManager.GetContainerStatuses(request). GetContainerStatuses()[0]; NUnit.Framework.Assert.AreEqual(ContainerState.Running, containerStatus.GetState( )); }
public ContainerManagementProtocol Run() { Configuration conf = new Configuration(); YarnRPC rpc = YarnRPC.Create(conf); IPEndPoint containerManagerBindAddress = NetUtils.CreateSocketAddrForHost("127.0.0.1" , 12345); return((ContainerManagementProtocol)rpc.GetProxy(typeof(ContainerManagementProtocol ), containerManagerBindAddress, conf)); }
public static Token NewContainerToken(NodeId nodeId, byte[] password, ContainerTokenIdentifier tokenIdentifier) { // RPC layer client expects ip:port as service for tokens IPEndPoint addr = NetUtils.CreateSocketAddrForHost(nodeId.GetHost(), nodeId.GetPort ()); // NOTE: use SecurityUtil.setTokenService if this becomes a "real" token Token containerToken = NewToken <Token>(tokenIdentifier.GetBytes(), ContainerTokenIdentifier .Kind.ToString(), password, SecurityUtil.BuildTokenService(addr).ToString()); return(containerToken); }
public static Org.Apache.Hadoop.Yarn.Api.Records.Token NewInstance(byte[] password , NMTokenIdentifier identifier) { NodeId nodeId = identifier.GetNodeId(); // RPC layer client expects ip:port as service for tokens IPEndPoint addr = NetUtils.CreateSocketAddrForHost(nodeId.GetHost(), nodeId.GetPort ()); Org.Apache.Hadoop.Yarn.Api.Records.Token nmToken = Org.Apache.Hadoop.Yarn.Api.Records.Token .NewInstance(identifier.GetBytes(), NMTokenIdentifier.Kind.ToString(), password, SecurityUtil.BuildTokenService(addr).ToString()); return(nmToken); }
// Protected. For tests. protected internal virtual ContainerManagementProtocol GetContainerMgrProxy(ContainerId containerId) { NodeId node = masterContainer.GetNodeId(); IPEndPoint containerManagerConnectAddress = NetUtils.CreateSocketAddrForHost(node .GetHost(), node.GetPort()); YarnRPC rpc = GetYarnRPC(); UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(containerId .GetApplicationAttemptId().ToString()); string user = rmContext.GetRMApps()[containerId.GetApplicationAttemptId().GetApplicationId ()].GetUser(); Token token = rmContext.GetNMTokenSecretManager().CreateNMToken(containerId.GetApplicationAttemptId (), node, user); currentUser.AddToken(ConverterUtils.ConvertFromYarn(token, containerManagerConnectAddress )); return(NMProxy.CreateNMProxy <ContainerManagementProtocol>(conf, currentUser, rpc, containerManagerConnectAddress)); }
SelectToken(URI nnUri, ICollection <Org.Apache.Hadoop.Security.Token.Token <object > > tokens, Configuration conf) { // this guesses the remote cluster's rpc service port. // the current token design assumes it's the same as the local cluster's // rpc port unless a config key is set. there should be a way to automatic // and correctly determine the value Text serviceName = SecurityUtil.BuildTokenService(nnUri); string nnServiceName = conf.Get(ServiceNameKey + serviceName); int nnRpcPort = NameNode.DefaultPort; if (nnServiceName != null) { nnRpcPort = NetUtils.CreateSocketAddr(nnServiceName, nnRpcPort).Port; } // use original hostname from the uri to avoid unintentional host resolving serviceName = SecurityUtil.BuildTokenService(NetUtils.CreateSocketAddrForHost(nnUri .GetHost(), nnRpcPort)); return(SelectToken(serviceName, tokens)); }
// check: // 1-4) combinations of host and port // this will construct a socket addr, verify all the fields, build the // service to verify the use_ip setting is honored, set the token service // based on addr and verify the token service is set correctly, decode // the token service and ensure all the fields of the decoded addr match private void VerifyServiceAddr(string host, string ip) { IPEndPoint addr; int port = 123; // test host, port tuple //LOG.info("test tuple ("+host+","+port+")"); addr = NetUtils.CreateSocketAddrForHost(host, port); VerifyAddress(addr, host, ip, port); // test authority with no default port //LOG.info("test authority '"+host+":"+port+"'"); addr = NetUtils.CreateSocketAddr(host + ":" + port); VerifyAddress(addr, host, ip, port); // test authority with a default port, make sure default isn't used //LOG.info("test authority '"+host+":"+port+"' with ignored default port"); addr = NetUtils.CreateSocketAddr(host + ":" + port, port + 1); VerifyAddress(addr, host, ip, port); // test host-only authority, using port as default port //LOG.info("test host:"+host+" port:"+port); addr = NetUtils.CreateSocketAddr(host, port); VerifyAddress(addr, host, ip, port); }
/// <exception cref="System.Exception"/> public static void Main(string[] args) { Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler ()); Log.Debug("Child starting"); JobConf job = new JobConf(MRJobConfig.JobConfFile); // Initing with our JobConf allows us to avoid loading confs twice Limits.Init(job); UserGroupInformation.SetConfiguration(job); string host = args[0]; int port = System.Convert.ToInt32(args[1]); IPEndPoint address = NetUtils.CreateSocketAddrForHost(host, port); TaskAttemptID firstTaskid = ((TaskAttemptID)TaskAttemptID.ForName(args[2])); long jvmIdLong = long.Parse(args[3]); JVMId jvmId = new JVMId(((JobID)firstTaskid.GetJobID()), firstTaskid.GetTaskType( ) == TaskType.Map, jvmIdLong); // initialize metrics DefaultMetricsSystem.Initialize(StringUtils.Camelize(firstTaskid.GetTaskType().ToString ()) + "Task"); // Security framework already loaded the tokens into current ugi Credentials credentials = UserGroupInformation.GetCurrentUser().GetCredentials(); Log.Info("Executing with tokens:"); foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in credentials.GetAllTokens ()) { Log.Info(token); } // Create TaskUmbilicalProtocol as actual task owner. UserGroupInformation taskOwner = UserGroupInformation.CreateRemoteUser(((JobID)firstTaskid .GetJobID()).ToString()); Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jt = TokenCache.GetJobToken (credentials); SecurityUtil.SetTokenService(jt, address); taskOwner.AddToken(jt); TaskUmbilicalProtocol umbilical = taskOwner.DoAs(new _PrivilegedExceptionAction_108 (address, job)); // report non-pid to application master JvmContext context = new JvmContext(jvmId, "-1000"); Log.Debug("PID: " + Sharpen.Runtime.GetEnv()["JVM_PID"]); Task task = null; UserGroupInformation childUGI = null; ScheduledExecutorService logSyncer = null; try { int idleLoopCount = 0; JvmTask myTask = null; // poll for new task for (int idle = 0; null == myTask; ++idle) { long sleepTimeMilliSecs = Math.Min(idle * 500, 1500); Log.Info("Sleeping for " + sleepTimeMilliSecs + "ms before retrying again. Got null now." ); TimeUnit.Milliseconds.Sleep(sleepTimeMilliSecs); myTask = umbilical.GetTask(context); } if (myTask.ShouldDie()) { return; } task = myTask.GetTask(); YarnChild.taskid = task.GetTaskID(); // Create the job-conf and set credentials ConfigureTask(job, task, credentials, jt); // Initiate Java VM metrics JvmMetrics.InitSingleton(jvmId.ToString(), job.GetSessionId()); childUGI = UserGroupInformation.CreateRemoteUser(Runtime.Getenv(ApplicationConstants.Environment .User.ToString())); // Add tokens to new user so that it may execute its task correctly. childUGI.AddCredentials(credentials); // set job classloader if configured before invoking the task MRApps.SetJobClassLoader(job); logSyncer = TaskLog.CreateLogSyncer(); // Create a final reference to the task for the doAs block Task taskFinal = task; childUGI.DoAs(new _PrivilegedExceptionAction_158(taskFinal, job, umbilical)); } catch (FSError e) { // use job-specified working directory // run the task Log.Fatal("FSError from child", e); if (!ShutdownHookManager.Get().IsShutdownInProgress()) { umbilical.FsError(taskid, e.Message); } } catch (Exception exception) { Log.Warn("Exception running child : " + StringUtils.StringifyException(exception) ); try { if (task != null) { // do cleanup for the task if (childUGI == null) { // no need to job into doAs block task.TaskCleanup(umbilical); } else { Task taskFinal = task; childUGI.DoAs(new _PrivilegedExceptionAction_183(taskFinal, umbilical)); } } } catch (Exception e) { Log.Info("Exception cleaning up: " + StringUtils.StringifyException(e)); } // Report back any failures, for diagnostic purposes if (taskid != null) { if (!ShutdownHookManager.Get().IsShutdownInProgress()) { umbilical.FatalError(taskid, StringUtils.StringifyException(exception)); } } } catch (Exception throwable) { Log.Fatal("Error running child : " + StringUtils.StringifyException(throwable)); if (taskid != null) { if (!ShutdownHookManager.Get().IsShutdownInProgress()) { Exception tCause = throwable.InnerException; string cause = tCause == null ? throwable.Message : StringUtils.StringifyException (tCause); umbilical.FatalError(taskid, cause); } } } finally { RPC.StopProxy(umbilical); DefaultMetricsSystem.Shutdown(); TaskLog.SyncLogsShutdown(logSyncer); } }
/// <exception cref="System.IO.IOException"/> private MRClientProtocol GetProxy() { if (realProxy != null) { return(realProxy); } // Possibly allow nulls through the PB tunnel, otherwise deal with an exception // and redirect to the history server. ApplicationReport application = null; try { application = rm.GetApplicationReport(appId); } catch (ApplicationNotFoundException) { application = null; } catch (YarnException e2) { throw new IOException(e2); } if (application != null) { trackingUrl = application.GetTrackingUrl(); } IPEndPoint serviceAddr = null; while (application == null || YarnApplicationState.Running == application.GetYarnApplicationState ()) { if (application == null) { Log.Info("Could not get Job info from RM for job " + jobId + ". Redirecting to job history server." ); return(CheckAndGetHSProxy(null, JobState.New)); } try { if (application.GetHost() == null || string.Empty.Equals(application.GetHost())) { Log.Debug("AM not assigned to Job. Waiting to get the AM ..."); Sharpen.Thread.Sleep(2000); Log.Debug("Application state is " + application.GetYarnApplicationState()); application = rm.GetApplicationReport(appId); continue; } else { if (Unavailable.Equals(application.GetHost())) { if (!amAclDisabledStatusLogged) { Log.Info("Job " + jobId + " is running, but the host is unknown." + " Verify user has VIEW_JOB access." ); amAclDisabledStatusLogged = true; } return(GetNotRunningJob(application, JobState.Running)); } } if (!conf.GetBoolean(MRJobConfig.JobAmAccessDisabled, false)) { UserGroupInformation newUgi = UserGroupInformation.CreateRemoteUser(UserGroupInformation .GetCurrentUser().GetUserName()); serviceAddr = NetUtils.CreateSocketAddrForHost(application.GetHost(), application .GetRpcPort()); if (UserGroupInformation.IsSecurityEnabled()) { Token clientToAMToken = application.GetClientToAMToken(); Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> token = ConverterUtils .ConvertFromYarn(clientToAMToken, serviceAddr); newUgi.AddToken(token); } Log.Debug("Connecting to " + serviceAddr); IPEndPoint finalServiceAddr = serviceAddr; realProxy = newUgi.DoAs(new _PrivilegedExceptionAction_202(this, finalServiceAddr )); } else { if (!amAclDisabledStatusLogged) { Log.Info("Network ACL closed to AM for job " + jobId + ". Not going to try to reach the AM." ); amAclDisabledStatusLogged = true; } return(GetNotRunningJob(null, JobState.Running)); } return(realProxy); } catch (IOException) { //possibly the AM has crashed //there may be some time before AM is restarted //keep retrying by getting the address from RM Log.Info("Could not connect to " + serviceAddr + ". Waiting for getting the latest AM address..." ); try { Sharpen.Thread.Sleep(2000); } catch (Exception e1) { Log.Warn("getProxy() call interruped", e1); throw new YarnRuntimeException(e1); } try { application = rm.GetApplicationReport(appId); } catch (YarnException e1) { throw new IOException(e1); } if (application == null) { Log.Info("Could not get Job info from RM for job " + jobId + ". Redirecting to job history server." ); return(CheckAndGetHSProxy(null, JobState.Running)); } } catch (Exception e) { Log.Warn("getProxy() call interruped", e); throw new YarnRuntimeException(e); } catch (YarnException e) { throw new IOException(e); } } string user = application.GetUser(); if (user == null) { throw new IOException("User is not set in the application report"); } if (application.GetYarnApplicationState() == YarnApplicationState.New || application .GetYarnApplicationState() == YarnApplicationState.NewSaving || application.GetYarnApplicationState () == YarnApplicationState.Submitted || application.GetYarnApplicationState() == YarnApplicationState.Accepted) { realProxy = null; return(GetNotRunningJob(application, JobState.New)); } if (application.GetYarnApplicationState() == YarnApplicationState.Failed) { realProxy = null; return(GetNotRunningJob(application, JobState.Failed)); } if (application.GetYarnApplicationState() == YarnApplicationState.Killed) { realProxy = null; return(GetNotRunningJob(application, JobState.Killed)); } //History server can serve a job only if application //succeeded. if (application.GetYarnApplicationState() == YarnApplicationState.Finished) { Log.Info("Application state is completed. FinalApplicationStatus=" + application. GetFinalApplicationStatus().ToString() + ". Redirecting to job history server"); realProxy = CheckAndGetHSProxy(application, JobState.Succeeded); } return(realProxy); }