public static ApplicationSubmissionContext NewApplicationSubmissionContext(ApplicationId applicationId, string applicationName, string queue, Priority priority, ContainerLaunchContext amContainer, bool isUnmanagedAM, bool cancelTokensWhenComplete, int maxAppAttempts , Resource resource, string applicationType) { ApplicationSubmissionContext context = recordFactory.NewRecordInstance <ApplicationSubmissionContext >(); context.SetApplicationId(applicationId); context.SetApplicationName(applicationName); context.SetQueue(queue); context.SetPriority(priority); context.SetAMContainerSpec(amContainer); context.SetUnmanagedAM(isUnmanagedAM); context.SetCancelTokensWhenComplete(cancelTokensWhenComplete); context.SetMaxAppAttempts(maxAppAttempts); context.SetResource(resource); context.SetApplicationType(applicationType); return(context); }
public virtual void Setup() { // start minicluster conf = new YarnConfiguration(); yarnCluster = new MiniYARNCluster(typeof(TestAMRMClient).FullName, nodeCount, 1, 1); yarnCluster.Init(conf); yarnCluster.Start(); NUnit.Framework.Assert.IsNotNull(yarnCluster); NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnCluster.GetServiceState ()); // start rm client yarnClient = (YarnClientImpl)YarnClient.CreateYarnClient(); yarnClient.Init(conf); yarnClient.Start(); NUnit.Framework.Assert.IsNotNull(yarnClient); NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnClient.GetServiceState ()); // get node info nodeReports = yarnClient.GetNodeReports(NodeState.Running); // submit new app ApplicationSubmissionContext appContext = yarnClient.CreateApplication().GetApplicationSubmissionContext (); ApplicationId appId = appContext.GetApplicationId(); // set the application name appContext.SetApplicationName("Test"); // Set the priority for the application master Priority pri = Priority.NewInstance(0); appContext.SetPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.SetQueue("default"); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext>(); appContext.SetAMContainerSpec(amContainer); // unmanaged AM appContext.SetUnmanagedAM(true); // Create the request to send to the applications manager SubmitApplicationRequest appRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <SubmitApplicationRequest>(); appRequest.SetApplicationSubmissionContext(appContext); // Submit the application to the applications manager yarnClient.SubmitApplication(appContext); // wait for app to start int iterationsLeft = 30; RMAppAttempt appAttempt = null; while (iterationsLeft > 0) { ApplicationReport appReport = yarnClient.GetApplicationReport(appId); if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted) { attemptId = appReport.GetCurrentApplicationAttemptId(); appAttempt = yarnCluster.GetResourceManager().GetRMContext().GetRMApps()[attemptId .GetApplicationId()].GetCurrentAppAttempt(); while (true) { if (appAttempt.GetAppAttemptState() == RMAppAttemptState.Launched) { break; } } break; } Sleep(1000); --iterationsLeft; } if (iterationsLeft == 0) { NUnit.Framework.Assert.Fail("Application hasn't bee started"); } // Just dig into the ResourceManager and get the AMRMToken just for the sake // of testing. UserGroupInformation.SetLoginUser(UserGroupInformation.CreateRemoteUser(UserGroupInformation .GetCurrentUser().GetUserName())); UserGroupInformation.GetCurrentUser().AddToken(appAttempt.GetAMRMToken()); //creating an instance NMTokenCase nmTokenCache = new NMTokenCache(); // start am rm client rmClient = (AMRMClientImpl <AMRMClient.ContainerRequest>)AMRMClient.CreateAMRMClient <AMRMClient.ContainerRequest>(); //setting an instance NMTokenCase rmClient.SetNMTokenCache(nmTokenCache); rmClient.Init(conf); rmClient.Start(); NUnit.Framework.Assert.IsNotNull(rmClient); NUnit.Framework.Assert.AreEqual(Service.STATE.Started, rmClient.GetServiceState() ); // start am nm client nmClient = (NMClientImpl)NMClient.CreateNMClient(); //propagating the AMRMClient NMTokenCache instance nmClient.SetNMTokenCache(rmClient.GetNMTokenCache()); nmClient.Init(conf); nmClient.Start(); NUnit.Framework.Assert.IsNotNull(nmClient); NUnit.Framework.Assert.AreEqual(Service.STATE.Started, nmClient.GetServiceState() ); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual bool Run() { Log.Info("Starting Client"); // Connect to ResourceManager rmClient.Start(); try { // Create launch context for app master Log.Info("Setting up application submission context for ASM"); ApplicationSubmissionContext appContext = rmClient.CreateApplication().GetApplicationSubmissionContext (); ApplicationId appId = appContext.GetApplicationId(); // set the application name appContext.SetApplicationName(appName); // Set the priority for the application master Priority pri = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Priority>(); pri.SetPriority(amPriority); appContext.SetPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.SetQueue(amQueue); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext>(); appContext.SetAMContainerSpec(amContainer); // unmanaged AM appContext.SetUnmanagedAM(true); Log.Info("Setting unmanaged AM"); // Submit the application to the applications manager Log.Info("Submitting application to ASM"); rmClient.SubmitApplication(appContext); ApplicationReport appReport = MonitorApplication(appId, EnumSet.Of(YarnApplicationState .Accepted, YarnApplicationState.Killed, YarnApplicationState.Failed, YarnApplicationState .Finished)); if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted) { // Monitor the application attempt to wait for launch state ApplicationAttemptReport attemptReport = MonitorCurrentAppAttempt(appId, YarnApplicationAttemptState .Launched); ApplicationAttemptId attemptId = attemptReport.GetApplicationAttemptId(); Log.Info("Launching AM with application attempt id " + attemptId); // launch AM LaunchAM(attemptId); // Monitor the application for end state appReport = MonitorApplication(appId, EnumSet.Of(YarnApplicationState.Killed, YarnApplicationState .Failed, YarnApplicationState.Finished)); } YarnApplicationState appState = appReport.GetYarnApplicationState(); FinalApplicationStatus appStatus = appReport.GetFinalApplicationStatus(); Log.Info("App ended with state: " + appReport.GetYarnApplicationState() + " and status: " + appStatus); bool success; if (YarnApplicationState.Finished == appState && FinalApplicationStatus.Succeeded == appStatus) { Log.Info("Application has completed successfully."); success = true; } else { Log.Info("Application did finished unsuccessfully." + " YarnState=" + appState.ToString () + ", FinalStatus=" + appStatus.ToString()); success = false; } return(success); } finally { rmClient.Stop(); } }
/// <exception cref="System.Exception"/> public virtual RMApp SubmitApp(int masterMemory, string name, string user, IDictionary <ApplicationAccessType, string> acls, bool unmanaged, string queue, int maxAppAttempts , Credentials ts, string appType, bool waitForAccepted, bool keepContainers, bool isAppIdProvided, ApplicationId applicationId, long attemptFailuresValidityInterval , LogAggregationContext logAggregationContext, bool cancelTokensWhenComplete) { ApplicationId appId = isAppIdProvided ? applicationId : null; ApplicationClientProtocol client = GetClientRMService(); if (!isAppIdProvided) { GetNewApplicationResponse resp = client.GetNewApplication(Org.Apache.Hadoop.Yarn.Util.Records .NewRecord <GetNewApplicationRequest>()); appId = resp.GetApplicationId(); } SubmitApplicationRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <SubmitApplicationRequest >(); ApplicationSubmissionContext sub = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord < ApplicationSubmissionContext>(); sub.SetKeepContainersAcrossApplicationAttempts(keepContainers); sub.SetApplicationId(appId); sub.SetApplicationName(name); sub.SetMaxAppAttempts(maxAppAttempts); if (unmanaged) { sub.SetUnmanagedAM(true); } if (queue != null) { sub.SetQueue(queue); } sub.SetApplicationType(appType); ContainerLaunchContext clc = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext >(); Resource capability = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Resource>(); capability.SetMemory(masterMemory); sub.SetResource(capability); clc.SetApplicationACLs(acls); if (ts != null && UserGroupInformation.IsSecurityEnabled()) { DataOutputBuffer dob = new DataOutputBuffer(); ts.WriteTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength()); clc.SetTokens(securityTokens); } sub.SetAMContainerSpec(clc); sub.SetAttemptFailuresValidityInterval(attemptFailuresValidityInterval); if (logAggregationContext != null) { sub.SetLogAggregationContext(logAggregationContext); } sub.SetCancelTokensWhenComplete(cancelTokensWhenComplete); req.SetApplicationSubmissionContext(sub); UserGroupInformation fakeUser = UserGroupInformation.CreateUserForTesting(user, new string[] { "someGroup" }); PrivilegedAction <SubmitApplicationResponse> action = new _PrivilegedAction_415(). SetClientReq(client, req); fakeUser.DoAs(action); // make sure app is immediately available after submit if (waitForAccepted) { WaitForState(appId, RMAppState.Accepted); } RMApp rmApp = GetRMContext().GetRMApps()[appId]; // unmanaged AM won't go to RMAppAttemptState.SCHEDULED. if (waitForAccepted && !unmanaged) { WaitForState(rmApp.GetCurrentAppAttempt().GetAppAttemptId(), RMAppAttemptState.Scheduled ); } return(rmApp); }