/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private RMAppImpl CreateAndPopulateNewRMApp(ApplicationSubmissionContext submissionContext
, long submitTime, string user, bool isRecovery)
{
ApplicationId applicationId = submissionContext.GetApplicationId();
ResourceRequest amReq = ValidateAndCreateResourceRequest(submissionContext, isRecovery
);
// Create RMApp
RMAppImpl application = new RMAppImpl(applicationId, rmContext, this.conf, submissionContext
.GetApplicationName(), user, submissionContext.GetQueue(), submissionContext, this
.scheduler, this.masterService, submitTime, submissionContext.GetApplicationType
(), submissionContext.GetApplicationTags(), amReq);
// Concurrent app submissions with same applicationId will fail here
// Concurrent app submissions with different applicationIds will not
// influence each other
if (rmContext.GetRMApps().PutIfAbsent(applicationId, application) != null)
{
string message = "Application with id " + applicationId + " is already present! Cannot add a duplicate!";
Log.Warn(message);
throw new YarnException(message);
}
// Inform the ACLs Manager
this.applicationACLsManager.AddApplication(applicationId, submissionContext.GetAMContainerSpec
().GetApplicationACLs());
string appViewACLs = submissionContext.GetAMContainerSpec().GetApplicationACLs()[
ApplicationAccessType.ViewApp];
rmContext.GetSystemMetricsPublisher().AppACLsUpdated(application, appViewACLs, Runtime
.CurrentTimeMillis());
return(application);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.InvalidResourceRequestException
/// "/>
private ResourceRequest ValidateAndCreateResourceRequest(ApplicationSubmissionContext
submissionContext, bool isRecovery)
{
// Validation of the ApplicationSubmissionContext needs to be completed
// here. Only those fields that are dependent on RM's configuration are
// checked here as they have to be validated whether they are part of new
// submission or just being recovered.
// Check whether AM resource requirements are within required limits
if (!submissionContext.GetUnmanagedAM())
{
ResourceRequest amReq = submissionContext.GetAMContainerResourceRequest();
if (amReq == null)
{
amReq = BuilderUtils.NewResourceRequest(RMAppAttemptImpl.AmContainerPriority, ResourceRequest
.Any, submissionContext.GetResource(), 1);
}
// set label expression for AM container
if (null == amReq.GetNodeLabelExpression())
{
amReq.SetNodeLabelExpression(submissionContext.GetNodeLabelExpression());
}
try
{
SchedulerUtils.NormalizeAndValidateRequest(amReq, scheduler.GetMaximumResourceCapability
(), submissionContext.GetQueue(), scheduler, isRecovery, rmContext);
}
catch (InvalidResourceRequestException e)
{
Log.Warn("RM app submission failed in validating AM resource request" + " for application "
+ submissionContext.GetApplicationId(), e);
throw;
}
SchedulerUtils.NormalizeRequest(amReq, scheduler.GetResourceCalculator(), scheduler
.GetClusterResource(), scheduler.GetMinimumResourceCapability(), scheduler.GetMaximumResourceCapability
(), scheduler.GetMinimumResourceCapability());
return(amReq);
}
return(null);
}
/// <exception cref="System.IO.IOException"/>
public virtual ApplicationSubmissionContext CreateApplicationSubmissionContext(Configuration
jobConf, string jobSubmitDir, Credentials ts)
{
ApplicationId applicationId = resMgrDelegate.GetApplicationId();
// Setup resource requirements
Resource capability = recordFactory.NewRecordInstance <Resource>();
capability.SetMemory(conf.GetInt(MRJobConfig.MrAmVmemMb, MRJobConfig.DefaultMrAmVmemMb
));
capability.SetVirtualCores(conf.GetInt(MRJobConfig.MrAmCpuVcores, MRJobConfig.DefaultMrAmCpuVcores
));
Log.Debug("AppMaster capability = " + capability);
// Setup LocalResources
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
Path jobConfPath = new Path(jobSubmitDir, MRJobConfig.JobConfFile);
URL yarnUrlForJobSubmitDir = ConverterUtils.GetYarnUrlFromPath(defaultFileContext
.GetDefaultFileSystem().ResolvePath(defaultFileContext.MakeQualified(new Path(jobSubmitDir
))));
Log.Debug("Creating setup context, jobSubmitDir url is " + yarnUrlForJobSubmitDir
);
localResources[MRJobConfig.JobConfFile] = CreateApplicationResource(defaultFileContext
, jobConfPath, LocalResourceType.File);
if (jobConf.Get(MRJobConfig.Jar) != null)
{
Path jobJarPath = new Path(jobConf.Get(MRJobConfig.Jar));
LocalResource rc = CreateApplicationResource(FileContext.GetFileContext(jobJarPath
.ToUri(), jobConf), jobJarPath, LocalResourceType.Pattern);
string pattern = conf.GetPattern(JobContext.JarUnpackPattern, JobConf.UnpackJarPatternDefault
).Pattern();
rc.SetPattern(pattern);
localResources[MRJobConfig.JobJar] = rc;
}
else
{
// Job jar may be null. For e.g, for pipes, the job jar is the hadoop
// mapreduce jar itself which is already on the classpath.
Log.Info("Job jar is not present. " + "Not adding any jar to the list of resources."
);
}
// TODO gross hack
foreach (string s in new string[] { MRJobConfig.JobSplit, MRJobConfig.JobSplitMetainfo })
{
localResources[MRJobConfig.JobSubmitDir + "/" + s] = CreateApplicationResource(defaultFileContext
, new Path(jobSubmitDir, s), LocalResourceType.File);
}
// Setup security tokens
DataOutputBuffer dob = new DataOutputBuffer();
ts.WriteTokenStorageToStream(dob);
ByteBuffer securityTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
// Setup the command to run the AM
IList <string> vargs = new AList <string>(8);
vargs.AddItem(MRApps.CrossPlatformifyMREnv(jobConf, ApplicationConstants.Environment
.JavaHome) + "/bin/java");
Path amTmpDir = new Path(MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment
.Pwd), YarnConfiguration.DefaultContainerTempDir);
vargs.AddItem("-Djava.io.tmpdir=" + amTmpDir);
MRApps.AddLog4jSystemProperties(null, vargs, conf);
// Check for Java Lib Path usage in MAP and REDUCE configs
WarnForJavaLibPath(conf.Get(MRJobConfig.MapJavaOpts, string.Empty), "map", MRJobConfig
.MapJavaOpts, MRJobConfig.MapEnv);
WarnForJavaLibPath(conf.Get(MRJobConfig.MapredMapAdminJavaOpts, string.Empty), "map"
, MRJobConfig.MapredMapAdminJavaOpts, MRJobConfig.MapredAdminUserEnv);
WarnForJavaLibPath(conf.Get(MRJobConfig.ReduceJavaOpts, string.Empty), "reduce",
MRJobConfig.ReduceJavaOpts, MRJobConfig.ReduceEnv);
WarnForJavaLibPath(conf.Get(MRJobConfig.MapredReduceAdminJavaOpts, string.Empty),
"reduce", MRJobConfig.MapredReduceAdminJavaOpts, MRJobConfig.MapredAdminUserEnv);
// Add AM admin command opts before user command opts
// so that it can be overridden by user
string mrAppMasterAdminOptions = conf.Get(MRJobConfig.MrAmAdminCommandOpts, MRJobConfig
.DefaultMrAmAdminCommandOpts);
WarnForJavaLibPath(mrAppMasterAdminOptions, "app master", MRJobConfig.MrAmAdminCommandOpts
, MRJobConfig.MrAmAdminUserEnv);
vargs.AddItem(mrAppMasterAdminOptions);
// Add AM user command opts
string mrAppMasterUserOptions = conf.Get(MRJobConfig.MrAmCommandOpts, MRJobConfig
.DefaultMrAmCommandOpts);
WarnForJavaLibPath(mrAppMasterUserOptions, "app master", MRJobConfig.MrAmCommandOpts
, MRJobConfig.MrAmEnv);
vargs.AddItem(mrAppMasterUserOptions);
if (jobConf.GetBoolean(MRJobConfig.MrAmProfile, MRJobConfig.DefaultMrAmProfile))
{
string profileParams = jobConf.Get(MRJobConfig.MrAmProfileParams, MRJobConfig.DefaultTaskProfileParams
);
if (profileParams != null)
{
vargs.AddItem(string.Format(profileParams, ApplicationConstants.LogDirExpansionVar
+ Path.Separator + TaskLog.LogName.Profile));
}
}
vargs.AddItem(MRJobConfig.ApplicationMasterClass);
vargs.AddItem("1>" + ApplicationConstants.LogDirExpansionVar + Path.Separator + ApplicationConstants
.Stdout);
vargs.AddItem("2>" + ApplicationConstants.LogDirExpansionVar + Path.Separator + ApplicationConstants
.Stderr);
Vector <string> vargsFinal = new Vector <string>(8);
// Final command
StringBuilder mergedCommand = new StringBuilder();
foreach (CharSequence str in vargs)
{
mergedCommand.Append(str).Append(" ");
}
vargsFinal.AddItem(mergedCommand.ToString());
Log.Debug("Command to launch container for ApplicationMaster is : " + mergedCommand
);
// Setup the CLASSPATH in environment
// i.e. add { Hadoop jars, job jar, CWD } to classpath.
IDictionary <string, string> environment = new Dictionary <string, string>();
MRApps.SetClasspath(environment, conf);
// Shell
environment[ApplicationConstants.Environment.Shell.ToString()] = conf.Get(MRJobConfig
.MapredAdminUserShell, MRJobConfig.DefaultShell);
// Add the container working directory at the front of LD_LIBRARY_PATH
MRApps.AddToEnvironment(environment, ApplicationConstants.Environment.LdLibraryPath
.ToString(), MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment
.Pwd), conf);
// Setup the environment variables for Admin first
MRApps.SetEnvFromInputString(environment, conf.Get(MRJobConfig.MrAmAdminUserEnv),
conf);
// Setup the environment variables (LD_LIBRARY_PATH, etc)
MRApps.SetEnvFromInputString(environment, conf.Get(MRJobConfig.MrAmEnv), conf);
// Parse distributed cache
MRApps.SetupDistributedCache(jobConf, localResources);
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>(2);
acls[ApplicationAccessType.ViewApp] = jobConf.Get(MRJobConfig.JobAclViewJob, MRJobConfig
.DefaultJobAclViewJob);
acls[ApplicationAccessType.ModifyApp] = jobConf.Get(MRJobConfig.JobAclModifyJob,
MRJobConfig.DefaultJobAclModifyJob);
// Setup ContainerLaunchContext for AM container
ContainerLaunchContext amContainer = ContainerLaunchContext.NewInstance(localResources
, environment, vargsFinal, null, securityTokens, acls);
ICollection <string> tagsFromConf = jobConf.GetTrimmedStringCollection(MRJobConfig
.JobTags);
// Set up the ApplicationSubmissionContext
ApplicationSubmissionContext appContext = recordFactory.NewRecordInstance <ApplicationSubmissionContext
>();
appContext.SetApplicationId(applicationId);
// ApplicationId
appContext.SetQueue(jobConf.Get(JobContext.QueueName, YarnConfiguration.DefaultQueueName
));
// Queue name
// add reservationID if present
ReservationId reservationID = null;
try
{
reservationID = ReservationId.ParseReservationId(jobConf.Get(JobContext.ReservationId
));
}
catch (FormatException)
{
// throw exception as reservationid as is invalid
string errMsg = "Invalid reservationId: " + jobConf.Get(JobContext.ReservationId)
+ " specified for the app: " + applicationId;
Log.Warn(errMsg);
throw new IOException(errMsg);
}
if (reservationID != null)
{
appContext.SetReservationID(reservationID);
Log.Info("SUBMITTING ApplicationSubmissionContext app:" + applicationId + " to queue:"
+ appContext.GetQueue() + " with reservationId:" + appContext.GetReservationID(
));
}
appContext.SetApplicationName(jobConf.Get(JobContext.JobName, YarnConfiguration.DefaultApplicationName
));
// Job name
appContext.SetCancelTokensWhenComplete(conf.GetBoolean(MRJobConfig.JobCancelDelegationToken
, true));
appContext.SetAMContainerSpec(amContainer);
// AM Container
appContext.SetMaxAppAttempts(conf.GetInt(MRJobConfig.MrAmMaxAttempts, MRJobConfig
.DefaultMrAmMaxAttempts));
appContext.SetResource(capability);
appContext.SetApplicationType(MRJobConfig.MrApplicationType);
if (tagsFromConf != null && !tagsFromConf.IsEmpty())
{
appContext.SetApplicationTags(new HashSet <string>(tagsFromConf));
}
return(appContext);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
protected internal override void SubmitApplication(ApplicationSubmissionContext submissionContext
, long submitTime, string user)
{
//Do nothing, just add the application to RMContext
RMAppImpl application = new RMAppImpl(submissionContext.GetApplicationId(), this.
rmContext, this.conf, submissionContext.GetApplicationName(), user, submissionContext
.GetQueue(), submissionContext, this.rmContext.GetScheduler(), this.rmContext.GetApplicationMasterService
(), submitTime, submissionContext.GetApplicationType(), submissionContext.GetApplicationTags
(), null);
this.rmContext.GetRMApps()[submissionContext.GetApplicationId()] = application;
}
