/// <exception cref="System.Exception"/> private ApplicationId SubmitAppAndGetAppId(AccessControlList viewACL, AccessControlList modifyACL) { SubmitApplicationRequest submitRequest = recordFactory.NewRecordInstance <SubmitApplicationRequest >(); ApplicationSubmissionContext context = recordFactory.NewRecordInstance <ApplicationSubmissionContext >(); ApplicationId applicationId = rmClient.GetNewApplication(recordFactory.NewRecordInstance <GetNewApplicationRequest>()).GetApplicationId(); context.SetApplicationId(applicationId); IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType , string>(); acls[ApplicationAccessType.ViewApp] = viewACL.GetAclString(); acls[ApplicationAccessType.ModifyApp] = modifyACL.GetAclString(); ContainerLaunchContext amContainer = recordFactory.NewRecordInstance <ContainerLaunchContext >(); Resource resource = BuilderUtils.NewResource(1024, 1); context.SetResource(resource); amContainer.SetApplicationACLs(acls); context.SetAMContainerSpec(amContainer); submitRequest.SetApplicationSubmissionContext(context); rmClient.SubmitApplication(submitRequest); resourceManager.WaitForState(applicationId, RMAppState.Accepted); return(applicationId); }
/// <exception cref="System.Exception"/> private ApplicationId SubmitAppAndGetAppId(string submitter, string queueName, bool setupACLs) { GetNewApplicationRequest newAppRequest = GetNewApplicationRequest.NewInstance(); ApplicationClientProtocol submitterClient = GetRMClientForUser(submitter); ApplicationId applicationId = submitterClient.GetNewApplication(newAppRequest).GetApplicationId (); Resource resource = BuilderUtils.NewResource(1024, 1); IDictionary <ApplicationAccessType, string> acls = CreateACLs(submitter, setupACLs ); ContainerLaunchContext amContainerSpec = ContainerLaunchContext.NewInstance(null, null, null, null, null, acls); ApplicationSubmissionContext appSubmissionContext = ApplicationSubmissionContext. NewInstance(applicationId, "applicationName", queueName, null, amContainerSpec, false, true, 1, resource, "applicationType"); appSubmissionContext.SetApplicationId(applicationId); appSubmissionContext.SetQueue(queueName); SubmitApplicationRequest submitRequest = SubmitApplicationRequest.NewInstance(appSubmissionContext ); submitterClient.SubmitApplication(submitRequest); resourceManager.WaitForState(applicationId, RMAppState.Accepted); return(applicationId); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual void Submit() { lock (this) { ApplicationSubmissionContext context = recordFactory.NewRecordInstance <ApplicationSubmissionContext >(); context.SetApplicationId(this.applicationId); context.SetQueue(this.queue); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext>(); context.SetAMContainerSpec(amContainer); context.SetResource(Resources.CreateResource(YarnConfiguration.DefaultRmSchedulerMinimumAllocationMb )); SubmitApplicationRequest request = recordFactory.NewRecordInstance <SubmitApplicationRequest >(); request.SetApplicationSubmissionContext(context); ResourceScheduler scheduler = resourceManager.GetResourceScheduler(); resourceManager.GetClientRMService().SubmitApplication(request); // Notify scheduler AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(this.applicationId , this.queue, "user"); scheduler.Handle(addAppEvent); AppAttemptAddedSchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent (this.applicationAttemptId, false); scheduler.Handle(addAttemptEvent); } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public override YarnClientApplication CreateApplication() { ApplicationSubmissionContext context = Records.NewRecord <ApplicationSubmissionContext >(); GetNewApplicationResponse newApp = GetNewApplication(); ApplicationId appId = newApp.GetApplicationId(); context.SetApplicationId(appId); return(new YarnClientApplication(newApp, context)); }
/// <exception cref="System.Exception"/> public virtual void TestRMAppSubmitMaxAppAttempts() { int[] globalMaxAppAttempts = new int[] { 10, 1 }; int[][] individualMaxAppAttempts = new int[][] { new int[] { 9, 10, 11, 0 }, new int[] { 1, 10, 0, -1 } }; int[][] expectedNums = new int[][] { new int[] { 9, 10, 10, 10 }, new int[] { 1, 1, 1, 1 } }; for (int i = 0; i < globalMaxAppAttempts.Length; ++i) { for (int j = 0; j < individualMaxAppAttempts.Length; ++j) { ResourceScheduler scheduler = MockResourceScheduler(); Configuration conf = new Configuration(); conf.SetInt(YarnConfiguration.RmAmMaxAttempts, globalMaxAppAttempts[i]); ApplicationMasterService masterService = new ApplicationMasterService(rmContext, scheduler); TestAppManager.TestRMAppManager appMonitor = new TestAppManager.TestRMAppManager( this, rmContext, new ClientToAMTokenSecretManagerInRM(), scheduler, masterService , new ApplicationACLsManager(conf), conf); ApplicationId appID = MockApps.NewAppID(i * 4 + j + 1); asContext.SetApplicationId(appID); if (individualMaxAppAttempts[i][j] != 0) { asContext.SetMaxAppAttempts(individualMaxAppAttempts[i][j]); } appMonitor.SubmitApplication(asContext, "test"); RMApp app = rmContext.GetRMApps()[appID]; NUnit.Framework.Assert.AreEqual("max application attempts doesn't match", expectedNums [i][j], app.GetMaxAppAttempts()); // wait for event to be processed int timeoutSecs = 0; while ((GetAppEventType() == RMAppEventType.Kill) && timeoutSecs++ < 20) { Sharpen.Thread.Sleep(1000); } SetAppEventType(RMAppEventType.Kill); } } }
public static ApplicationSubmissionContext NewApplicationSubmissionContext(ApplicationId applicationId, string applicationName, string queue, Priority priority, ContainerLaunchContext amContainer, bool isUnmanagedAM, bool cancelTokensWhenComplete, int maxAppAttempts , Resource resource, string applicationType) { ApplicationSubmissionContext context = recordFactory.NewRecordInstance <ApplicationSubmissionContext >(); context.SetApplicationId(applicationId); context.SetApplicationName(applicationName); context.SetQueue(queue); context.SetPriority(priority); context.SetAMContainerSpec(amContainer); context.SetUnmanagedAM(isUnmanagedAM); context.SetCancelTokensWhenComplete(cancelTokensWhenComplete); context.SetMaxAppAttempts(maxAppAttempts); context.SetResource(resource); context.SetApplicationType(applicationType); return(context); }
/// <exception cref="System.Exception"/> public virtual void TestSubmitApplicationOnHA() { ApplicationSubmissionContext appContext = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ApplicationSubmissionContext>(); appContext.SetApplicationId(cluster.CreateFakeAppId()); ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext>(); appContext.SetAMContainerSpec(amContainer); Resource capability = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Resource>(); capability.SetMemory(10); capability.SetVirtualCores(1); appContext.SetResource(capability); ApplicationId appId = client.SubmitApplication(appContext); NUnit.Framework.Assert.IsTrue(GetActiveRM().GetRMContext().GetRMApps().Contains(appId )); }
public virtual void SetUp() { long now = Runtime.CurrentTimeMillis(); rmContext = MockRMContext(1, now - 10); ResourceScheduler scheduler = MockResourceScheduler(); Configuration conf = new Configuration(); ApplicationMasterService masterService = new ApplicationMasterService(rmContext, scheduler); appMonitor = new TestAppManager.TestRMAppManager(this, rmContext, new ClientToAMTokenSecretManagerInRM (), scheduler, masterService, new ApplicationACLsManager(conf), conf); appId = MockApps.NewAppID(1); RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(null); asContext = recordFactory.NewRecordInstance <ApplicationSubmissionContext>(); asContext.SetApplicationId(appId); asContext.SetAMContainerSpec(MockContainerLaunchContext(recordFactory)); asContext.SetResource(MockResource()); SetupDispatcher(rmContext, conf); }
/// <exception cref="System.IO.IOException"/> public virtual ApplicationSubmissionContext CreateApplicationSubmissionContext(Configuration jobConf, string jobSubmitDir, Credentials ts) { ApplicationId applicationId = resMgrDelegate.GetApplicationId(); // Setup resource requirements Resource capability = recordFactory.NewRecordInstance <Resource>(); capability.SetMemory(conf.GetInt(MRJobConfig.MrAmVmemMb, MRJobConfig.DefaultMrAmVmemMb )); capability.SetVirtualCores(conf.GetInt(MRJobConfig.MrAmCpuVcores, MRJobConfig.DefaultMrAmCpuVcores )); Log.Debug("AppMaster capability = " + capability); // Setup LocalResources IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); Path jobConfPath = new Path(jobSubmitDir, MRJobConfig.JobConfFile); URL yarnUrlForJobSubmitDir = ConverterUtils.GetYarnUrlFromPath(defaultFileContext .GetDefaultFileSystem().ResolvePath(defaultFileContext.MakeQualified(new Path(jobSubmitDir )))); Log.Debug("Creating setup context, jobSubmitDir url is " + yarnUrlForJobSubmitDir ); localResources[MRJobConfig.JobConfFile] = CreateApplicationResource(defaultFileContext , jobConfPath, LocalResourceType.File); if (jobConf.Get(MRJobConfig.Jar) != null) { Path jobJarPath = new Path(jobConf.Get(MRJobConfig.Jar)); LocalResource rc = CreateApplicationResource(FileContext.GetFileContext(jobJarPath .ToUri(), jobConf), jobJarPath, LocalResourceType.Pattern); string pattern = conf.GetPattern(JobContext.JarUnpackPattern, JobConf.UnpackJarPatternDefault ).Pattern(); rc.SetPattern(pattern); localResources[MRJobConfig.JobJar] = rc; } else { // Job jar may be null. For e.g, for pipes, the job jar is the hadoop // mapreduce jar itself which is already on the classpath. Log.Info("Job jar is not present. " + "Not adding any jar to the list of resources." ); } // TODO gross hack foreach (string s in new string[] { MRJobConfig.JobSplit, MRJobConfig.JobSplitMetainfo }) { localResources[MRJobConfig.JobSubmitDir + "/" + s] = CreateApplicationResource(defaultFileContext , new Path(jobSubmitDir, s), LocalResourceType.File); } // Setup security tokens DataOutputBuffer dob = new DataOutputBuffer(); ts.WriteTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength()); // Setup the command to run the AM IList <string> vargs = new AList <string>(8); vargs.AddItem(MRApps.CrossPlatformifyMREnv(jobConf, ApplicationConstants.Environment .JavaHome) + "/bin/java"); Path amTmpDir = new Path(MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment .Pwd), YarnConfiguration.DefaultContainerTempDir); vargs.AddItem("-Djava.io.tmpdir=" + amTmpDir); MRApps.AddLog4jSystemProperties(null, vargs, conf); // Check for Java Lib Path usage in MAP and REDUCE configs WarnForJavaLibPath(conf.Get(MRJobConfig.MapJavaOpts, string.Empty), "map", MRJobConfig .MapJavaOpts, MRJobConfig.MapEnv); WarnForJavaLibPath(conf.Get(MRJobConfig.MapredMapAdminJavaOpts, string.Empty), "map" , MRJobConfig.MapredMapAdminJavaOpts, MRJobConfig.MapredAdminUserEnv); WarnForJavaLibPath(conf.Get(MRJobConfig.ReduceJavaOpts, string.Empty), "reduce", MRJobConfig.ReduceJavaOpts, MRJobConfig.ReduceEnv); WarnForJavaLibPath(conf.Get(MRJobConfig.MapredReduceAdminJavaOpts, string.Empty), "reduce", MRJobConfig.MapredReduceAdminJavaOpts, MRJobConfig.MapredAdminUserEnv); // Add AM admin command opts before user command opts // so that it can be overridden by user string mrAppMasterAdminOptions = conf.Get(MRJobConfig.MrAmAdminCommandOpts, MRJobConfig .DefaultMrAmAdminCommandOpts); WarnForJavaLibPath(mrAppMasterAdminOptions, "app master", MRJobConfig.MrAmAdminCommandOpts , MRJobConfig.MrAmAdminUserEnv); vargs.AddItem(mrAppMasterAdminOptions); // Add AM user command opts string mrAppMasterUserOptions = conf.Get(MRJobConfig.MrAmCommandOpts, MRJobConfig .DefaultMrAmCommandOpts); WarnForJavaLibPath(mrAppMasterUserOptions, "app master", MRJobConfig.MrAmCommandOpts , MRJobConfig.MrAmEnv); vargs.AddItem(mrAppMasterUserOptions); if (jobConf.GetBoolean(MRJobConfig.MrAmProfile, MRJobConfig.DefaultMrAmProfile)) { string profileParams = jobConf.Get(MRJobConfig.MrAmProfileParams, MRJobConfig.DefaultTaskProfileParams ); if (profileParams != null) { vargs.AddItem(string.Format(profileParams, ApplicationConstants.LogDirExpansionVar + Path.Separator + TaskLog.LogName.Profile)); } } vargs.AddItem(MRJobConfig.ApplicationMasterClass); vargs.AddItem("1>" + ApplicationConstants.LogDirExpansionVar + Path.Separator + ApplicationConstants .Stdout); vargs.AddItem("2>" + ApplicationConstants.LogDirExpansionVar + Path.Separator + ApplicationConstants .Stderr); Vector <string> vargsFinal = new Vector <string>(8); // Final command StringBuilder mergedCommand = new StringBuilder(); foreach (CharSequence str in vargs) { mergedCommand.Append(str).Append(" "); } vargsFinal.AddItem(mergedCommand.ToString()); Log.Debug("Command to launch container for ApplicationMaster is : " + mergedCommand ); // Setup the CLASSPATH in environment // i.e. add { Hadoop jars, job jar, CWD } to classpath. IDictionary <string, string> environment = new Dictionary <string, string>(); MRApps.SetClasspath(environment, conf); // Shell environment[ApplicationConstants.Environment.Shell.ToString()] = conf.Get(MRJobConfig .MapredAdminUserShell, MRJobConfig.DefaultShell); // Add the container working directory at the front of LD_LIBRARY_PATH MRApps.AddToEnvironment(environment, ApplicationConstants.Environment.LdLibraryPath .ToString(), MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment .Pwd), conf); // Setup the environment variables for Admin first MRApps.SetEnvFromInputString(environment, conf.Get(MRJobConfig.MrAmAdminUserEnv), conf); // Setup the environment variables (LD_LIBRARY_PATH, etc) MRApps.SetEnvFromInputString(environment, conf.Get(MRJobConfig.MrAmEnv), conf); // Parse distributed cache MRApps.SetupDistributedCache(jobConf, localResources); IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType , string>(2); acls[ApplicationAccessType.ViewApp] = jobConf.Get(MRJobConfig.JobAclViewJob, MRJobConfig .DefaultJobAclViewJob); acls[ApplicationAccessType.ModifyApp] = jobConf.Get(MRJobConfig.JobAclModifyJob, MRJobConfig.DefaultJobAclModifyJob); // Setup ContainerLaunchContext for AM container ContainerLaunchContext amContainer = ContainerLaunchContext.NewInstance(localResources , environment, vargsFinal, null, securityTokens, acls); ICollection <string> tagsFromConf = jobConf.GetTrimmedStringCollection(MRJobConfig .JobTags); // Set up the ApplicationSubmissionContext ApplicationSubmissionContext appContext = recordFactory.NewRecordInstance <ApplicationSubmissionContext >(); appContext.SetApplicationId(applicationId); // ApplicationId appContext.SetQueue(jobConf.Get(JobContext.QueueName, YarnConfiguration.DefaultQueueName )); // Queue name // add reservationID if present ReservationId reservationID = null; try { reservationID = ReservationId.ParseReservationId(jobConf.Get(JobContext.ReservationId )); } catch (FormatException) { // throw exception as reservationid as is invalid string errMsg = "Invalid reservationId: " + jobConf.Get(JobContext.ReservationId) + " specified for the app: " + applicationId; Log.Warn(errMsg); throw new IOException(errMsg); } if (reservationID != null) { appContext.SetReservationID(reservationID); Log.Info("SUBMITTING ApplicationSubmissionContext app:" + applicationId + " to queue:" + appContext.GetQueue() + " with reservationId:" + appContext.GetReservationID( )); } appContext.SetApplicationName(jobConf.Get(JobContext.JobName, YarnConfiguration.DefaultApplicationName )); // Job name appContext.SetCancelTokensWhenComplete(conf.GetBoolean(MRJobConfig.JobCancelDelegationToken , true)); appContext.SetAMContainerSpec(amContainer); // AM Container appContext.SetMaxAppAttempts(conf.GetInt(MRJobConfig.MrAmMaxAttempts, MRJobConfig .DefaultMrAmMaxAttempts)); appContext.SetResource(capability); appContext.SetApplicationType(MRJobConfig.MrApplicationType); if (tagsFromConf != null && !tagsFromConf.IsEmpty()) { appContext.SetApplicationTags(new HashSet <string>(tagsFromConf)); } return(appContext); }
/// <exception cref="System.Exception"/> public virtual RMApp SubmitApp(int masterMemory, string name, string user, IDictionary <ApplicationAccessType, string> acls, bool unmanaged, string queue, int maxAppAttempts , Credentials ts, string appType, bool waitForAccepted, bool keepContainers, bool isAppIdProvided, ApplicationId applicationId, long attemptFailuresValidityInterval , LogAggregationContext logAggregationContext, bool cancelTokensWhenComplete) { ApplicationId appId = isAppIdProvided ? applicationId : null; ApplicationClientProtocol client = GetClientRMService(); if (!isAppIdProvided) { GetNewApplicationResponse resp = client.GetNewApplication(Org.Apache.Hadoop.Yarn.Util.Records .NewRecord <GetNewApplicationRequest>()); appId = resp.GetApplicationId(); } SubmitApplicationRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <SubmitApplicationRequest >(); ApplicationSubmissionContext sub = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord < ApplicationSubmissionContext>(); sub.SetKeepContainersAcrossApplicationAttempts(keepContainers); sub.SetApplicationId(appId); sub.SetApplicationName(name); sub.SetMaxAppAttempts(maxAppAttempts); if (unmanaged) { sub.SetUnmanagedAM(true); } if (queue != null) { sub.SetQueue(queue); } sub.SetApplicationType(appType); ContainerLaunchContext clc = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext >(); Resource capability = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Resource>(); capability.SetMemory(masterMemory); sub.SetResource(capability); clc.SetApplicationACLs(acls); if (ts != null && UserGroupInformation.IsSecurityEnabled()) { DataOutputBuffer dob = new DataOutputBuffer(); ts.WriteTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength()); clc.SetTokens(securityTokens); } sub.SetAMContainerSpec(clc); sub.SetAttemptFailuresValidityInterval(attemptFailuresValidityInterval); if (logAggregationContext != null) { sub.SetLogAggregationContext(logAggregationContext); } sub.SetCancelTokensWhenComplete(cancelTokensWhenComplete); req.SetApplicationSubmissionContext(sub); UserGroupInformation fakeUser = UserGroupInformation.CreateUserForTesting(user, new string[] { "someGroup" }); PrivilegedAction <SubmitApplicationResponse> action = new _PrivilegedAction_415(). SetClientReq(client, req); fakeUser.DoAs(action); // make sure app is immediately available after submit if (waitForAccepted) { WaitForState(appId, RMAppState.Accepted); } RMApp rmApp = GetRMContext().GetRMApps()[appId]; // unmanaged AM won't go to RMAppAttemptState.SCHEDULED. if (waitForAccepted && !unmanaged) { WaitForState(rmApp.GetCurrentAppAttempt().GetAppAttemptId(), RMAppAttemptState.Scheduled ); } return(rmApp); }