/// <exception cref="System.Exception"/> private ApplicationId SubmitAppAndGetAppId(AccessControlList viewACL, AccessControlList modifyACL) { SubmitApplicationRequest submitRequest = recordFactory.NewRecordInstance <SubmitApplicationRequest >(); ApplicationSubmissionContext context = recordFactory.NewRecordInstance <ApplicationSubmissionContext >(); ApplicationId applicationId = rmClient.GetNewApplication(recordFactory.NewRecordInstance <GetNewApplicationRequest>()).GetApplicationId(); context.SetApplicationId(applicationId); IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType , string>(); acls[ApplicationAccessType.ViewApp] = viewACL.GetAclString(); acls[ApplicationAccessType.ModifyApp] = modifyACL.GetAclString(); ContainerLaunchContext amContainer = recordFactory.NewRecordInstance <ContainerLaunchContext >(); Resource resource = BuilderUtils.NewResource(1024, 1); context.SetResource(resource); amContainer.SetApplicationACLs(acls); context.SetAMContainerSpec(amContainer); submitRequest.SetApplicationSubmissionContext(context); rmClient.SubmitApplication(submitRequest); resourceManager.WaitForState(applicationId, RMAppState.Accepted); return(applicationId); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public override JobStatus SubmitJob(JobID jobId, string jobSubmitDir, Credentials ts) { AddHistoryToken(ts); // Construct necessary information to start the MR AM ApplicationSubmissionContext appContext = CreateApplicationSubmissionContext(conf , jobSubmitDir, ts); // Submit to ResourceManager try { ApplicationId applicationId = resMgrDelegate.SubmitApplication(appContext); ApplicationReport appMaster = resMgrDelegate.GetApplicationReport(applicationId); string diagnostics = (appMaster == null ? "application report is null" : appMaster .GetDiagnostics()); if (appMaster == null || appMaster.GetYarnApplicationState() == YarnApplicationState .Failed || appMaster.GetYarnApplicationState() == YarnApplicationState.Killed) { throw new IOException("Failed to run job : " + diagnostics); } return(clientCache.GetClient(jobId).GetJobStatus(jobId)); } catch (YarnException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual void Submit() { lock (this) { ApplicationSubmissionContext context = recordFactory.NewRecordInstance <ApplicationSubmissionContext >(); context.SetApplicationId(this.applicationId); context.SetQueue(this.queue); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext>(); context.SetAMContainerSpec(amContainer); context.SetResource(Resources.CreateResource(YarnConfiguration.DefaultRmSchedulerMinimumAllocationMb )); SubmitApplicationRequest request = recordFactory.NewRecordInstance <SubmitApplicationRequest >(); request.SetApplicationSubmissionContext(context); ResourceScheduler scheduler = resourceManager.GetResourceScheduler(); resourceManager.GetClientRMService().SubmitApplication(request); // Notify scheduler AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(this.applicationId , this.queue, "user"); scheduler.Handle(addAppEvent); AppAttemptAddedSchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent (this.applicationAttemptId, false); scheduler.Handle(addAttemptEvent); } }
/// <exception cref="System.Exception"/> private ApplicationId SubmitAppAndGetAppId(string submitter, string queueName, bool setupACLs) { GetNewApplicationRequest newAppRequest = GetNewApplicationRequest.NewInstance(); ApplicationClientProtocol submitterClient = GetRMClientForUser(submitter); ApplicationId applicationId = submitterClient.GetNewApplication(newAppRequest).GetApplicationId (); Resource resource = BuilderUtils.NewResource(1024, 1); IDictionary <ApplicationAccessType, string> acls = CreateACLs(submitter, setupACLs ); ContainerLaunchContext amContainerSpec = ContainerLaunchContext.NewInstance(null, null, null, null, null, acls); ApplicationSubmissionContext appSubmissionContext = ApplicationSubmissionContext. NewInstance(applicationId, "applicationName", queueName, null, amContainerSpec, false, true, 1, resource, "applicationType"); appSubmissionContext.SetApplicationId(applicationId); appSubmissionContext.SetQueue(queueName); SubmitApplicationRequest submitRequest = SubmitApplicationRequest.NewInstance(appSubmissionContext ); submitterClient.SubmitApplication(submitRequest); resourceManager.WaitForState(applicationId, RMAppState.Accepted); return(applicationId); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> private RMAppImpl CreateAndPopulateNewRMApp(ApplicationSubmissionContext submissionContext , long submitTime, string user, bool isRecovery) { ApplicationId applicationId = submissionContext.GetApplicationId(); ResourceRequest amReq = ValidateAndCreateResourceRequest(submissionContext, isRecovery ); // Create RMApp RMAppImpl application = new RMAppImpl(applicationId, rmContext, this.conf, submissionContext .GetApplicationName(), user, submissionContext.GetQueue(), submissionContext, this .scheduler, this.masterService, submitTime, submissionContext.GetApplicationType (), submissionContext.GetApplicationTags(), amReq); // Concurrent app submissions with same applicationId will fail here // Concurrent app submissions with different applicationIds will not // influence each other if (rmContext.GetRMApps().PutIfAbsent(applicationId, application) != null) { string message = "Application with id " + applicationId + " is already present! Cannot add a duplicate!"; Log.Warn(message); throw new YarnException(message); } // Inform the ACLs Manager this.applicationACLsManager.AddApplication(applicationId, submissionContext.GetAMContainerSpec ().GetApplicationACLs()); string appViewACLs = submissionContext.GetAMContainerSpec().GetApplicationACLs()[ ApplicationAccessType.ViewApp]; rmContext.GetSystemMetricsPublisher().AppACLsUpdated(application, appViewACLs, Runtime .CurrentTimeMillis()); return(application); }
/// <exception cref="System.Exception"/> public virtual void TestWarnCommandOpts() { Logger logger = Logger.GetLogger(typeof(YARNRunner)); ByteArrayOutputStream bout = new ByteArrayOutputStream(); Layout layout = new SimpleLayout(); Appender appender = new WriterAppender(layout, bout); logger.AddAppender(appender); JobConf jobConf = new JobConf(); jobConf.Set(MRJobConfig.MrAmAdminCommandOpts, "-Djava.net.preferIPv4Stack=true -Djava.library.path=foo" ); jobConf.Set(MRJobConfig.MrAmCommandOpts, "-Xmx1024m -Djava.library.path=bar"); YARNRunner yarnRunner = new YARNRunner(jobConf); ApplicationSubmissionContext submissionContext = BuildSubmitContext(yarnRunner, jobConf ); string logMsg = bout.ToString(); NUnit.Framework.Assert.IsTrue(logMsg.Contains("WARN - Usage of -Djava.library.path in " + "yarn.app.mapreduce.am.admin-command-opts can cause programs to no " + "longer function if hadoop native libraries are used. These values " + "should be set as part of the LD_LIBRARY_PATH in the app master JVM " + "env using yarn.app.mapreduce.am.admin.user.env config settings." )); NUnit.Framework.Assert.IsTrue(logMsg.Contains("WARN - Usage of -Djava.library.path in " + "yarn.app.mapreduce.am.command-opts can cause programs to no longer " + "function if hadoop native libraries are used. These values should " + "be set as part of the LD_LIBRARY_PATH in the app master JVM env " + "using yarn.app.mapreduce.am.env config settings." )); }
public virtual void TestAMStandardEnv() { string AdminLibPath = "foo"; string UserLibPath = "bar"; string UserShell = "shell"; JobConf jobConf = new JobConf(); jobConf.Set(MRJobConfig.MrAmAdminUserEnv, "LD_LIBRARY_PATH=" + AdminLibPath); jobConf.Set(MRJobConfig.MrAmEnv, "LD_LIBRARY_PATH=" + UserLibPath); jobConf.Set(MRJobConfig.MapredAdminUserShell, UserShell); YARNRunner yarnRunner = new YARNRunner(jobConf); ApplicationSubmissionContext appSubCtx = BuildSubmitContext(yarnRunner, jobConf); // make sure PWD is first in the lib path ContainerLaunchContext clc = appSubCtx.GetAMContainerSpec(); IDictionary <string, string> env = clc.GetEnvironment(); string libPath = env[ApplicationConstants.Environment.LdLibraryPath.ToString()]; NUnit.Framework.Assert.IsNotNull("LD_LIBRARY_PATH not set", libPath); string cps = jobConf.GetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, MRConfig .DefaultMapreduceAppSubmissionCrossPlatform) ? ApplicationConstants.ClassPathSeparator : FilePath.pathSeparator; NUnit.Framework.Assert.AreEqual("Bad AM LD_LIBRARY_PATH setting", MRApps.CrossPlatformifyMREnv (conf, ApplicationConstants.Environment.Pwd) + cps + AdminLibPath + cps + UserLibPath , libPath); // make sure SHELL is set string shell = env[ApplicationConstants.Environment.Shell.ToString()]; NUnit.Framework.Assert.IsNotNull("SHELL not set", shell); NUnit.Framework.Assert.AreEqual("Bad SHELL setting", UserShell, shell); }
public SchedulerApplicationAttempt(ApplicationAttemptId applicationAttemptId, string user, Queue queue, ActiveUsersManager activeUsersManager, RMContext rmContext) { // This pendingRelease is used in work-preserving recovery scenario to keep // track of the AM's outstanding release requests. RM on recovery could // receive the release request form AM before it receives the container status // from NM for recovery. In this case, the to-be-recovered containers reported // by NM should not be recovered. // Time of the last container scheduled at the current allowed level Preconditions.CheckNotNull(rmContext, "RMContext should not be null"); this.rmContext = rmContext; this.appSchedulingInfo = new AppSchedulingInfo(applicationAttemptId, user, queue, activeUsersManager, rmContext.GetEpoch()); this.queue = queue; this.pendingRelease = new HashSet <ContainerId>(); this.attemptId = applicationAttemptId; if (rmContext.GetRMApps() != null && rmContext.GetRMApps().Contains(applicationAttemptId .GetApplicationId())) { ApplicationSubmissionContext appSubmissionContext = rmContext.GetRMApps()[applicationAttemptId .GetApplicationId()].GetApplicationSubmissionContext(); if (appSubmissionContext != null) { unmanagedAM = appSubmissionContext.GetUnmanagedAM(); this.logAggregationContext = appSubmissionContext.GetLogAggregationContext(); } } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> private void Launch() { Connect(); ContainerId masterContainerID = masterContainer.GetId(); ApplicationSubmissionContext applicationContext = application.GetSubmissionContext (); Log.Info("Setting up container " + masterContainer + " for AM " + application.GetAppAttemptId ()); ContainerLaunchContext launchContext = CreateAMContainerLaunchContext(applicationContext , masterContainerID); StartContainerRequest scRequest = StartContainerRequest.NewInstance(launchContext , masterContainer.GetContainerToken()); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); StartContainersResponse response = containerMgrProxy.StartContainers(allRequests); if (response.GetFailedRequests() != null && response.GetFailedRequests().Contains (masterContainerID)) { Exception t = response.GetFailedRequests()[masterContainerID].DeSerialize(); ParseAndThrowException(t); } else { Log.Info("Done launching container " + masterContainer + " for AM " + application .GetAppAttemptId()); } }
/// <exception cref="System.Exception"/> public virtual void TestAMAdminCommandOpts() { JobConf jobConf = new JobConf(); jobConf.Set(MRJobConfig.MrAmAdminCommandOpts, "-Djava.net.preferIPv4Stack=true"); jobConf.Set(MRJobConfig.MrAmCommandOpts, "-Xmx1024m"); YARNRunner yarnRunner = new YARNRunner(jobConf); ApplicationSubmissionContext submissionContext = BuildSubmitContext(yarnRunner, jobConf ); ContainerLaunchContext containerSpec = submissionContext.GetAMContainerSpec(); IList <string> commands = containerSpec.GetCommands(); int index = 0; int adminIndex = 0; int adminPos = -1; int userIndex = 0; int userPos = -1; int tmpDirPos = -1; foreach (string command in commands) { if (command != null) { NUnit.Framework.Assert.IsFalse("Profiler should be disabled by default", command. Contains(ProfileParams)); adminPos = command.IndexOf("-Djava.net.preferIPv4Stack=true"); if (adminPos >= 0) { adminIndex = index; } userPos = command.IndexOf("-Xmx1024m"); if (userPos >= 0) { userIndex = index; } tmpDirPos = command.IndexOf("-Djava.io.tmpdir="); } index++; } // Check java.io.tmpdir opts are set in the commands NUnit.Framework.Assert.IsTrue("java.io.tmpdir is not set for AM", tmpDirPos > 0); // Check both admin java opts and user java opts are in the commands NUnit.Framework.Assert.IsTrue("AM admin command opts not in the commands.", adminPos > 0); NUnit.Framework.Assert.IsTrue("AM user command opts not in the commands.", userPos > 0); // Check the admin java opts is before user java opts in the commands if (adminIndex == userIndex) { NUnit.Framework.Assert.IsTrue("AM admin command opts is after user command opts." , adminPos < userPos); } else { NUnit.Framework.Assert.IsTrue("AM admin command opts is after user command opts." , adminIndex < userIndex); } }
public static SubmitApplicationRequest NewInstance(ApplicationSubmissionContext context ) { SubmitApplicationRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord < SubmitApplicationRequest>(); request.SetApplicationSubmissionContext(context); return(request); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public override YarnClientApplication CreateApplication() { ApplicationSubmissionContext context = Records.NewRecord <ApplicationSubmissionContext >(); GetNewApplicationResponse newApp = GetNewApplication(); ApplicationId appId = newApp.GetApplicationId(); context.SetApplicationId(appId); return(new YarnClientApplication(newApp, context)); }
public override void SetApplicationSubmissionContext(ApplicationSubmissionContext context) { MaybeInitBuilder(); if (context == null) { builder.ClearApplicationSubmissionContext(); } this.applicationSubmissionContext = context; }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> protected internal override void SubmitApplication(ApplicationSubmissionContext submissionContext , long submitTime, string user) { //Do nothing, just add the application to RMContext RMAppImpl application = new RMAppImpl(submissionContext.GetApplicationId(), this. rmContext, this.conf, submissionContext.GetApplicationName(), user, submissionContext .GetQueue(), submissionContext, this.rmContext.GetScheduler(), this.rmContext.GetApplicationMasterService (), submitTime, submissionContext.GetApplicationType(), submissionContext.GetApplicationTags (), null); this.rmContext.GetRMApps()[submissionContext.GetApplicationId()] = application; }
/// <exception cref="System.Exception"/> protected internal virtual void RecoverApplication(ApplicationStateData appState, RMStateStore.RMState rmState) { ApplicationSubmissionContext appContext = appState.GetApplicationSubmissionContext (); ApplicationId appId = appContext.GetApplicationId(); // create and recover app. RMAppImpl application = CreateAndPopulateNewRMApp(appContext, appState.GetSubmitTime (), appState.GetUser(), true); application.Handle(new RMAppRecoverEvent(appId, rmState)); }
// TODO: Don't create again and again. /// <exception cref="System.IO.IOException"/> private ContainerLaunchContext CreateAMContainerLaunchContext(ApplicationSubmissionContext applicationMasterContext, ContainerId containerID) { // Construct the actual Container ContainerLaunchContext container = applicationMasterContext.GetAMContainerSpec(); Log.Info("Command to launch container " + containerID + " : " + StringUtils.ArrayToString (Sharpen.Collections.ToArray(container.GetCommands(), new string[0]))); // Finalize the container SetupTokens(container, containerID); return(container); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual JobID GetNewJobID() { try { this.application = client.CreateApplication().GetApplicationSubmissionContext(); this.applicationId = this.application.GetApplicationId(); return(TypeConverter.FromYarn(applicationId)); } catch (YarnException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> protected internal virtual Credentials ParseCredentials(ApplicationSubmissionContext application) { Credentials credentials = new Credentials(); DataInputByteBuffer dibb = new DataInputByteBuffer(); ByteBuffer tokens = application.GetAMContainerSpec().GetTokens(); if (tokens != null) { dibb.Reset(tokens); credentials.ReadTokenStorageStream(dibb); tokens.Rewind(); } return(credentials); }
public override ApplicationSubmissionContext GetApplicationSubmissionContext() { YarnServiceProtos.SubmitApplicationRequestProtoOrBuilder p = viaProto ? proto : builder; if (this.applicationSubmissionContext != null) { return(this.applicationSubmissionContext); } if (!p.HasApplicationSubmissionContext()) { return(null); } this.applicationSubmissionContext = ConvertFromProtoFormat(p.GetApplicationSubmissionContext ()); return(this.applicationSubmissionContext); }
public override ApplicationSubmissionContext GetApplicationSubmissionContext() { YarnServerResourceManagerRecoveryProtos.ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder; if (applicationSubmissionContext != null) { return(applicationSubmissionContext); } if (!p.HasApplicationSubmissionContext()) { return(null); } applicationSubmissionContext = new ApplicationSubmissionContextPBImpl(p.GetApplicationSubmissionContext ()); return(applicationSubmissionContext); }
public static ApplicationStateData NewInstance(long submitTime, long startTime, string user, ApplicationSubmissionContext submissionContext, RMAppState state, string diagnostics, long finishTime) { ApplicationStateData appState = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ApplicationStateData >(); appState.SetSubmitTime(submitTime); appState.SetStartTime(startTime); appState.SetUser(user); appState.SetApplicationSubmissionContext(submissionContext); appState.SetState(state); appState.SetDiagnostics(diagnostics); appState.SetFinishTime(finishTime); return(appState); }
protected internal virtual void CreateApplicationWithAMResource(ApplicationAttemptId attId, string queue, string user, Resource amResource) { RMContext rmContext = resourceManager.GetRMContext(); RMApp rmApp = new RMAppImpl(attId.GetApplicationId(), rmContext, conf, null, null , null, ApplicationSubmissionContext.NewInstance(null, null, null, null, null, false , false, 0, amResource, null), null, null, 0, null, null, null); rmContext.GetRMApps()[attId.GetApplicationId()] = rmApp; AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(attId.GetApplicationId (), queue, user); scheduler.Handle(appAddedEvent); AppAttemptAddedSchedulerEvent attempAddedEvent = new AppAttemptAddedSchedulerEvent (attId, false); scheduler.Handle(attempAddedEvent); }
/// <exception cref="System.Exception"/> public virtual void TestSubmitApplicationOnHA() { ApplicationSubmissionContext appContext = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ApplicationSubmissionContext>(); appContext.SetApplicationId(cluster.CreateFakeAppId()); ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext>(); appContext.SetAMContainerSpec(amContainer); Resource capability = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Resource>(); capability.SetMemory(10); capability.SetVirtualCores(1); appContext.SetResource(capability); ApplicationId appId = client.SubmitApplication(appContext); NUnit.Framework.Assert.IsTrue(GetActiveRM().GetRMContext().GetRMApps().Contains(appId )); }
public static ApplicationSubmissionContext NewApplicationSubmissionContext(ApplicationId applicationId, string applicationName, string queue, Priority priority, ContainerLaunchContext amContainer, bool isUnmanagedAM, bool cancelTokensWhenComplete, int maxAppAttempts , Resource resource, string applicationType) { ApplicationSubmissionContext context = recordFactory.NewRecordInstance <ApplicationSubmissionContext >(); context.SetApplicationId(applicationId); context.SetApplicationName(applicationName); context.SetQueue(queue); context.SetPriority(priority); context.SetAMContainerSpec(amContainer); context.SetUnmanagedAM(isUnmanagedAM); context.SetCancelTokensWhenComplete(cancelTokensWhenComplete); context.SetMaxAppAttempts(maxAppAttempts); context.SetResource(resource); context.SetApplicationType(applicationType); return(context); }
public virtual void SetUp() { long now = Runtime.CurrentTimeMillis(); rmContext = MockRMContext(1, now - 10); ResourceScheduler scheduler = MockResourceScheduler(); Configuration conf = new Configuration(); ApplicationMasterService masterService = new ApplicationMasterService(rmContext, scheduler); appMonitor = new TestAppManager.TestRMAppManager(this, rmContext, new ClientToAMTokenSecretManagerInRM (), scheduler, masterService, new ApplicationACLsManager(conf), conf); appId = MockApps.NewAppID(1); RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(null); asContext = recordFactory.NewRecordInstance <ApplicationSubmissionContext>(); asContext.SetApplicationId(appId); asContext.SetAMContainerSpec(MockContainerLaunchContext(recordFactory)); asContext.SetResource(MockResource()); SetupDispatcher(rmContext, conf); }
protected override void SetUp() { resourceMgrDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); conf = new YarnConfiguration(); conf.Set(YarnConfiguration.RmPrincipal, "mapred/host@REALM"); clientCache = new ClientCache(conf, resourceMgrDelegate); clientCache = Org.Mockito.Mockito.Spy(clientCache); yarnRunner = new YARNRunner(conf, resourceMgrDelegate, clientCache); yarnRunner = Org.Mockito.Mockito.Spy(yarnRunner); submissionContext = Org.Mockito.Mockito.Mock <ApplicationSubmissionContext>(); Org.Mockito.Mockito.DoAnswer(new _Answer_146(this)).When(yarnRunner).CreateApplicationSubmissionContext (Matchers.Any <Configuration>(), Matchers.Any <string>(), Matchers.Any <Credentials >()); appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 1); jobId = TypeConverter.FromYarn(appId); if (testWorkDir.Exists()) { FileContext.GetLocalFSFileContext().Delete(new Path(testWorkDir.ToString()), true ); } testWorkDir.Mkdirs(); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.InvalidResourceRequestException /// "/> private ResourceRequest ValidateAndCreateResourceRequest(ApplicationSubmissionContext submissionContext, bool isRecovery) { // Validation of the ApplicationSubmissionContext needs to be completed // here. Only those fields that are dependent on RM's configuration are // checked here as they have to be validated whether they are part of new // submission or just being recovered. // Check whether AM resource requirements are within required limits if (!submissionContext.GetUnmanagedAM()) { ResourceRequest amReq = submissionContext.GetAMContainerResourceRequest(); if (amReq == null) { amReq = BuilderUtils.NewResourceRequest(RMAppAttemptImpl.AmContainerPriority, ResourceRequest .Any, submissionContext.GetResource(), 1); } // set label expression for AM container if (null == amReq.GetNodeLabelExpression()) { amReq.SetNodeLabelExpression(submissionContext.GetNodeLabelExpression()); } try { SchedulerUtils.NormalizeAndValidateRequest(amReq, scheduler.GetMaximumResourceCapability (), submissionContext.GetQueue(), scheduler, isRecovery, rmContext); } catch (InvalidResourceRequestException e) { Log.Warn("RM app submission failed in validating AM resource request" + " for application " + submissionContext.GetApplicationId(), e); throw; } SchedulerUtils.NormalizeRequest(amReq, scheduler.GetResourceCalculator(), scheduler .GetClusterResource(), scheduler.GetMinimumResourceCapability(), scheduler.GetMaximumResourceCapability (), scheduler.GetMinimumResourceCapability()); return(amReq); } return(null); }
/// <exception cref="System.Exception"/> public virtual void TestAMProfiler() { JobConf jobConf = new JobConf(); jobConf.SetBoolean(MRJobConfig.MrAmProfile, true); YARNRunner yarnRunner = new YARNRunner(jobConf); ApplicationSubmissionContext submissionContext = BuildSubmitContext(yarnRunner, jobConf ); ContainerLaunchContext containerSpec = submissionContext.GetAMContainerSpec(); IList <string> commands = containerSpec.GetCommands(); foreach (string command in commands) { if (command != null) { if (command.Contains(ProfileParams)) { return; } } } throw new InvalidOperationException("Profiler opts not found!"); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> protected internal virtual void SubmitApplication(ApplicationSubmissionContext submissionContext , long submitTime, string user) { ApplicationId applicationId = submissionContext.GetApplicationId(); RMAppImpl application = CreateAndPopulateNewRMApp(submissionContext, submitTime, user, false); ApplicationId appId = submissionContext.GetApplicationId(); if (UserGroupInformation.IsSecurityEnabled()) { try { this.rmContext.GetDelegationTokenRenewer().AddApplicationAsync(appId, ParseCredentials (submissionContext), submissionContext.GetCancelTokensWhenComplete(), application .GetUser()); } catch (Exception e) { Log.Warn("Unable to parse credentials.", e); // Sending APP_REJECTED is fine, since we assume that the // RMApp is in NEW state and thus we haven't yet informed the // scheduler about the existence of the application System.Diagnostics.Debug.Assert(application.GetState() == RMAppState.New); this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppEvent(applicationId , RMAppEventType.AppRejected, e.Message)); throw RPCUtil.GetRemoteException(e); } } else { // Dispatcher is not yet started at this time, so these START events // enqueued should be guaranteed to be first processed when dispatcher // gets started. this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppEvent(applicationId , RMAppEventType.Start)); } }
public virtual void Setup() { // start minicluster conf = new YarnConfiguration(); yarnCluster = new MiniYARNCluster(typeof(TestAMRMClient).FullName, nodeCount, 1, 1); yarnCluster.Init(conf); yarnCluster.Start(); NUnit.Framework.Assert.IsNotNull(yarnCluster); NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnCluster.GetServiceState ()); // start rm client yarnClient = (YarnClientImpl)YarnClient.CreateYarnClient(); yarnClient.Init(conf); yarnClient.Start(); NUnit.Framework.Assert.IsNotNull(yarnClient); NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnClient.GetServiceState ()); // get node info nodeReports = yarnClient.GetNodeReports(NodeState.Running); // submit new app ApplicationSubmissionContext appContext = yarnClient.CreateApplication().GetApplicationSubmissionContext (); ApplicationId appId = appContext.GetApplicationId(); // set the application name appContext.SetApplicationName("Test"); // Set the priority for the application master Priority pri = Priority.NewInstance(0); appContext.SetPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.SetQueue("default"); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext>(); appContext.SetAMContainerSpec(amContainer); // unmanaged AM appContext.SetUnmanagedAM(true); // Create the request to send to the applications manager SubmitApplicationRequest appRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <SubmitApplicationRequest>(); appRequest.SetApplicationSubmissionContext(appContext); // Submit the application to the applications manager yarnClient.SubmitApplication(appContext); // wait for app to start int iterationsLeft = 30; RMAppAttempt appAttempt = null; while (iterationsLeft > 0) { ApplicationReport appReport = yarnClient.GetApplicationReport(appId); if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted) { attemptId = appReport.GetCurrentApplicationAttemptId(); appAttempt = yarnCluster.GetResourceManager().GetRMContext().GetRMApps()[attemptId .GetApplicationId()].GetCurrentAppAttempt(); while (true) { if (appAttempt.GetAppAttemptState() == RMAppAttemptState.Launched) { break; } } break; } Sleep(1000); --iterationsLeft; } if (iterationsLeft == 0) { NUnit.Framework.Assert.Fail("Application hasn't bee started"); } // Just dig into the ResourceManager and get the AMRMToken just for the sake // of testing. UserGroupInformation.SetLoginUser(UserGroupInformation.CreateRemoteUser(UserGroupInformation .GetCurrentUser().GetUserName())); UserGroupInformation.GetCurrentUser().AddToken(appAttempt.GetAMRMToken()); //creating an instance NMTokenCase nmTokenCache = new NMTokenCache(); // start am rm client rmClient = (AMRMClientImpl <AMRMClient.ContainerRequest>)AMRMClient.CreateAMRMClient <AMRMClient.ContainerRequest>(); //setting an instance NMTokenCase rmClient.SetNMTokenCache(nmTokenCache); rmClient.Init(conf); rmClient.Start(); NUnit.Framework.Assert.IsNotNull(rmClient); NUnit.Framework.Assert.AreEqual(Service.STATE.Started, rmClient.GetServiceState() ); // start am nm client nmClient = (NMClientImpl)NMClient.CreateNMClient(); //propagating the AMRMClient NMTokenCache instance nmClient.SetNMTokenCache(rmClient.GetNMTokenCache()); nmClient.Init(conf); nmClient.Start(); NUnit.Framework.Assert.IsNotNull(nmClient); NUnit.Framework.Assert.AreEqual(Service.STATE.Started, nmClient.GetServiceState() ); }