/// <exception cref="System.Exception"/>
private ApplicationId SubmitAppAndGetAppId(string submitter, string queueName, bool
setupACLs)
{
GetNewApplicationRequest newAppRequest = GetNewApplicationRequest.NewInstance();
ApplicationClientProtocol submitterClient = GetRMClientForUser(submitter);
ApplicationId applicationId = submitterClient.GetNewApplication(newAppRequest).GetApplicationId
();
Resource resource = BuilderUtils.NewResource(1024, 1);
IDictionary <ApplicationAccessType, string> acls = CreateACLs(submitter, setupACLs
);
ContainerLaunchContext amContainerSpec = ContainerLaunchContext.NewInstance(null,
null, null, null, null, acls);
ApplicationSubmissionContext appSubmissionContext = ApplicationSubmissionContext.
NewInstance(applicationId, "applicationName", queueName, null, amContainerSpec,
false, true, 1, resource, "applicationType");
appSubmissionContext.SetApplicationId(applicationId);
appSubmissionContext.SetQueue(queueName);
SubmitApplicationRequest submitRequest = SubmitApplicationRequest.NewInstance(appSubmissionContext
);
submitterClient.SubmitApplication(submitRequest);
resourceManager.WaitForState(applicationId, RMAppState.Accepted);
return(applicationId);
}
public virtual void TestApplicationRecovery()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.NmRecoveryEnabled, true);
conf.Set(YarnConfiguration.NmAddress, "localhost:1234");
conf.SetBoolean(YarnConfiguration.YarnAclEnable, true);
conf.Set(YarnConfiguration.YarnAdminAcl, "yarn_admin_user");
NMStateStoreService stateStore = new NMMemoryStateStoreService();
stateStore.Init(conf);
stateStore.Start();
Context context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf
), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore
);
ContainerManagerImpl cm = CreateContainerManager(context);
cm.Init(conf);
cm.Start();
// simulate registration with RM
MasterKey masterKey = new MasterKeyPBImpl();
masterKey.SetKeyId(123);
masterKey.SetBytes(ByteBuffer.Wrap(new byte[] { 123 }));
context.GetContainerTokenSecretManager().SetMasterKey(masterKey);
context.GetNMTokenSecretManager().SetMasterKey(masterKey);
// add an application by starting a container
string appUser = "******";
string modUser = "******";
string viewUser = "******";
string enemyUser = "******";
ApplicationId appId = ApplicationId.NewInstance(0, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
ContainerId cid = ContainerId.NewContainerId(attemptId, 1);
IDictionary <string, LocalResource> localResources = Collections.EmptyMap();
IDictionary <string, string> containerEnv = Sharpen.Collections.EmptyMap();
IList <string> containerCmds = Sharpen.Collections.EmptyList();
IDictionary <string, ByteBuffer> serviceData = Sharpen.Collections.EmptyMap();
Credentials containerCreds = new Credentials();
DataOutputBuffer dob = new DataOutputBuffer();
containerCreds.WriteTokenStorageToStream(dob);
ByteBuffer containerTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>();
acls[ApplicationAccessType.ModifyApp] = modUser;
acls[ApplicationAccessType.ViewApp] = viewUser;
ContainerLaunchContext clc = ContainerLaunchContext.NewInstance(localResources, containerEnv
, containerCmds, serviceData, containerTokens, acls);
// create the logAggregationContext
LogAggregationContext logAggregationContext = LogAggregationContext.NewInstance("includePattern"
, "excludePattern", "includePatternInRollingAggregation", "excludePatternInRollingAggregation"
);
StartContainersResponse startResponse = StartContainer(context, cm, cid, clc, logAggregationContext
);
NUnit.Framework.Assert.IsTrue(startResponse.GetFailedRequests().IsEmpty());
NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = context.GetApplications()[appId];
NUnit.Framework.Assert.IsNotNull(app);
WaitForAppState(app, ApplicationState.Initing);
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId));
// reset container manager and verify app recovered with proper acls
cm.Stop();
context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new
NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore);
cm = CreateContainerManager(context);
cm.Init(conf);
cm.Start();
NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count);
app = context.GetApplications()[appId];
NUnit.Framework.Assert.IsNotNull(app);
// check whether LogAggregationContext is recovered correctly
LogAggregationContext recovered = ((ApplicationImpl)app).GetLogAggregationContext
();
NUnit.Framework.Assert.IsNotNull(recovered);
NUnit.Framework.Assert.AreEqual(logAggregationContext.GetIncludePattern(), recovered
.GetIncludePattern());
NUnit.Framework.Assert.AreEqual(logAggregationContext.GetExcludePattern(), recovered
.GetExcludePattern());
NUnit.Framework.Assert.AreEqual(logAggregationContext.GetRolledLogsIncludePattern
(), recovered.GetRolledLogsIncludePattern());
NUnit.Framework.Assert.AreEqual(logAggregationContext.GetRolledLogsExcludePattern
(), recovered.GetRolledLogsExcludePattern());
WaitForAppState(app, ApplicationState.Initing);
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId));
// simulate application completion
IList <ApplicationId> finishedApps = new AList <ApplicationId>();
finishedApps.AddItem(appId);
cm.Handle(new CMgrCompletedAppsEvent(finishedApps, CMgrCompletedAppsEvent.Reason.
ByResourcemanager));
WaitForAppState(app, ApplicationState.ApplicationResourcesCleaningup);
// restart and verify app is marked for finishing
cm.Stop();
context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new
NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore);
cm = CreateContainerManager(context);
cm.Init(conf);
cm.Start();
NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count);
app = context.GetApplications()[appId];
NUnit.Framework.Assert.IsNotNull(app);
WaitForAppState(app, ApplicationState.ApplicationResourcesCleaningup);
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId));
// simulate log aggregation completion
app.Handle(new ApplicationEvent(app.GetAppId(), ApplicationEventType.ApplicationResourcesCleanedup
));
NUnit.Framework.Assert.AreEqual(app.GetApplicationState(), ApplicationState.Finished
);
app.Handle(new ApplicationEvent(app.GetAppId(), ApplicationEventType.ApplicationLogHandlingFinished
));
// restart and verify app is no longer present after recovery
cm.Stop();
context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new
NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore);
cm = CreateContainerManager(context);
cm.Init(conf);
cm.Start();
NUnit.Framework.Assert.IsTrue(context.GetApplications().IsEmpty());
cm.Stop();
}
/// <exception cref="System.IO.IOException"/>
public virtual ApplicationSubmissionContext CreateApplicationSubmissionContext(Configuration
jobConf, string jobSubmitDir, Credentials ts)
{
ApplicationId applicationId = resMgrDelegate.GetApplicationId();
// Setup resource requirements
Resource capability = recordFactory.NewRecordInstance <Resource>();
capability.SetMemory(conf.GetInt(MRJobConfig.MrAmVmemMb, MRJobConfig.DefaultMrAmVmemMb
));
capability.SetVirtualCores(conf.GetInt(MRJobConfig.MrAmCpuVcores, MRJobConfig.DefaultMrAmCpuVcores
));
Log.Debug("AppMaster capability = " + capability);
// Setup LocalResources
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
Path jobConfPath = new Path(jobSubmitDir, MRJobConfig.JobConfFile);
URL yarnUrlForJobSubmitDir = ConverterUtils.GetYarnUrlFromPath(defaultFileContext
.GetDefaultFileSystem().ResolvePath(defaultFileContext.MakeQualified(new Path(jobSubmitDir
))));
Log.Debug("Creating setup context, jobSubmitDir url is " + yarnUrlForJobSubmitDir
);
localResources[MRJobConfig.JobConfFile] = CreateApplicationResource(defaultFileContext
, jobConfPath, LocalResourceType.File);
if (jobConf.Get(MRJobConfig.Jar) != null)
{
Path jobJarPath = new Path(jobConf.Get(MRJobConfig.Jar));
LocalResource rc = CreateApplicationResource(FileContext.GetFileContext(jobJarPath
.ToUri(), jobConf), jobJarPath, LocalResourceType.Pattern);
string pattern = conf.GetPattern(JobContext.JarUnpackPattern, JobConf.UnpackJarPatternDefault
).Pattern();
rc.SetPattern(pattern);
localResources[MRJobConfig.JobJar] = rc;
}
else
{
// Job jar may be null. For e.g, for pipes, the job jar is the hadoop
// mapreduce jar itself which is already on the classpath.
Log.Info("Job jar is not present. " + "Not adding any jar to the list of resources."
);
}
// TODO gross hack
foreach (string s in new string[] { MRJobConfig.JobSplit, MRJobConfig.JobSplitMetainfo })
{
localResources[MRJobConfig.JobSubmitDir + "/" + s] = CreateApplicationResource(defaultFileContext
, new Path(jobSubmitDir, s), LocalResourceType.File);
}
// Setup security tokens
DataOutputBuffer dob = new DataOutputBuffer();
ts.WriteTokenStorageToStream(dob);
ByteBuffer securityTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
// Setup the command to run the AM
IList <string> vargs = new AList <string>(8);
vargs.AddItem(MRApps.CrossPlatformifyMREnv(jobConf, ApplicationConstants.Environment
.JavaHome) + "/bin/java");
Path amTmpDir = new Path(MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment
.Pwd), YarnConfiguration.DefaultContainerTempDir);
vargs.AddItem("-Djava.io.tmpdir=" + amTmpDir);
MRApps.AddLog4jSystemProperties(null, vargs, conf);
// Check for Java Lib Path usage in MAP and REDUCE configs
WarnForJavaLibPath(conf.Get(MRJobConfig.MapJavaOpts, string.Empty), "map", MRJobConfig
.MapJavaOpts, MRJobConfig.MapEnv);
WarnForJavaLibPath(conf.Get(MRJobConfig.MapredMapAdminJavaOpts, string.Empty), "map"
, MRJobConfig.MapredMapAdminJavaOpts, MRJobConfig.MapredAdminUserEnv);
WarnForJavaLibPath(conf.Get(MRJobConfig.ReduceJavaOpts, string.Empty), "reduce",
MRJobConfig.ReduceJavaOpts, MRJobConfig.ReduceEnv);
WarnForJavaLibPath(conf.Get(MRJobConfig.MapredReduceAdminJavaOpts, string.Empty),
"reduce", MRJobConfig.MapredReduceAdminJavaOpts, MRJobConfig.MapredAdminUserEnv);
// Add AM admin command opts before user command opts
// so that it can be overridden by user
string mrAppMasterAdminOptions = conf.Get(MRJobConfig.MrAmAdminCommandOpts, MRJobConfig
.DefaultMrAmAdminCommandOpts);
WarnForJavaLibPath(mrAppMasterAdminOptions, "app master", MRJobConfig.MrAmAdminCommandOpts
, MRJobConfig.MrAmAdminUserEnv);
vargs.AddItem(mrAppMasterAdminOptions);
// Add AM user command opts
string mrAppMasterUserOptions = conf.Get(MRJobConfig.MrAmCommandOpts, MRJobConfig
.DefaultMrAmCommandOpts);
WarnForJavaLibPath(mrAppMasterUserOptions, "app master", MRJobConfig.MrAmCommandOpts
, MRJobConfig.MrAmEnv);
vargs.AddItem(mrAppMasterUserOptions);
if (jobConf.GetBoolean(MRJobConfig.MrAmProfile, MRJobConfig.DefaultMrAmProfile))
{
string profileParams = jobConf.Get(MRJobConfig.MrAmProfileParams, MRJobConfig.DefaultTaskProfileParams
);
if (profileParams != null)
{
vargs.AddItem(string.Format(profileParams, ApplicationConstants.LogDirExpansionVar
+ Path.Separator + TaskLog.LogName.Profile));
}
}
vargs.AddItem(MRJobConfig.ApplicationMasterClass);
vargs.AddItem("1>" + ApplicationConstants.LogDirExpansionVar + Path.Separator + ApplicationConstants
.Stdout);
vargs.AddItem("2>" + ApplicationConstants.LogDirExpansionVar + Path.Separator + ApplicationConstants
.Stderr);
Vector <string> vargsFinal = new Vector <string>(8);
// Final command
StringBuilder mergedCommand = new StringBuilder();
foreach (CharSequence str in vargs)
{
mergedCommand.Append(str).Append(" ");
}
vargsFinal.AddItem(mergedCommand.ToString());
Log.Debug("Command to launch container for ApplicationMaster is : " + mergedCommand
);
// Setup the CLASSPATH in environment
// i.e. add { Hadoop jars, job jar, CWD } to classpath.
IDictionary <string, string> environment = new Dictionary <string, string>();
MRApps.SetClasspath(environment, conf);
// Shell
environment[ApplicationConstants.Environment.Shell.ToString()] = conf.Get(MRJobConfig
.MapredAdminUserShell, MRJobConfig.DefaultShell);
// Add the container working directory at the front of LD_LIBRARY_PATH
MRApps.AddToEnvironment(environment, ApplicationConstants.Environment.LdLibraryPath
.ToString(), MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment
.Pwd), conf);
// Setup the environment variables for Admin first
MRApps.SetEnvFromInputString(environment, conf.Get(MRJobConfig.MrAmAdminUserEnv),
conf);
// Setup the environment variables (LD_LIBRARY_PATH, etc)
MRApps.SetEnvFromInputString(environment, conf.Get(MRJobConfig.MrAmEnv), conf);
// Parse distributed cache
MRApps.SetupDistributedCache(jobConf, localResources);
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>(2);
acls[ApplicationAccessType.ViewApp] = jobConf.Get(MRJobConfig.JobAclViewJob, MRJobConfig
.DefaultJobAclViewJob);
acls[ApplicationAccessType.ModifyApp] = jobConf.Get(MRJobConfig.JobAclModifyJob,
MRJobConfig.DefaultJobAclModifyJob);
// Setup ContainerLaunchContext for AM container
ContainerLaunchContext amContainer = ContainerLaunchContext.NewInstance(localResources
, environment, vargsFinal, null, securityTokens, acls);
ICollection <string> tagsFromConf = jobConf.GetTrimmedStringCollection(MRJobConfig
.JobTags);
// Set up the ApplicationSubmissionContext
ApplicationSubmissionContext appContext = recordFactory.NewRecordInstance <ApplicationSubmissionContext
>();
appContext.SetApplicationId(applicationId);
// ApplicationId
appContext.SetQueue(jobConf.Get(JobContext.QueueName, YarnConfiguration.DefaultQueueName
));
// Queue name
// add reservationID if present
ReservationId reservationID = null;
try
{
reservationID = ReservationId.ParseReservationId(jobConf.Get(JobContext.ReservationId
));
}
catch (FormatException)
{
// throw exception as reservationid as is invalid
string errMsg = "Invalid reservationId: " + jobConf.Get(JobContext.ReservationId)
+ " specified for the app: " + applicationId;
Log.Warn(errMsg);
throw new IOException(errMsg);
}
if (reservationID != null)
{
appContext.SetReservationID(reservationID);
Log.Info("SUBMITTING ApplicationSubmissionContext app:" + applicationId + " to queue:"
+ appContext.GetQueue() + " with reservationId:" + appContext.GetReservationID(
));
}
appContext.SetApplicationName(jobConf.Get(JobContext.JobName, YarnConfiguration.DefaultApplicationName
));
// Job name
appContext.SetCancelTokensWhenComplete(conf.GetBoolean(MRJobConfig.JobCancelDelegationToken
, true));
appContext.SetAMContainerSpec(amContainer);
// AM Container
appContext.SetMaxAppAttempts(conf.GetInt(MRJobConfig.MrAmMaxAttempts, MRJobConfig
.DefaultMrAmMaxAttempts));
appContext.SetResource(capability);
appContext.SetApplicationType(MRJobConfig.MrApplicationType);
if (tagsFromConf != null && !tagsFromConf.IsEmpty())
{
appContext.SetApplicationTags(new HashSet <string>(tagsFromConf));
}
return(appContext);
}
/// <summary>Main run function for the client</summary>
/// <returns>true if application completed successfully</returns>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual bool Run()
{
Log.Info("Running Client");
yarnClient.Start();
YarnClusterMetrics clusterMetrics = yarnClient.GetYarnClusterMetrics();
Log.Info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics
.GetNumNodeManagers());
IList <NodeReport> clusterNodeReports = yarnClient.GetNodeReports(NodeState.Running
);
Log.Info("Got Cluster node info from ASM");
foreach (NodeReport node in clusterNodeReports)
{
Log.Info("Got node report from ASM for" + ", nodeId=" + node.GetNodeId() + ", nodeAddress"
+ node.GetHttpAddress() + ", nodeRackName" + node.GetRackName() + ", nodeNumContainers"
+ node.GetNumContainers());
}
QueueInfo queueInfo = yarnClient.GetQueueInfo(this.amQueue);
Log.Info("Queue info" + ", queueName=" + queueInfo.GetQueueName() + ", queueCurrentCapacity="
+ queueInfo.GetCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.GetMaximumCapacity
() + ", queueApplicationCount=" + queueInfo.GetApplications().Count + ", queueChildQueueCount="
+ queueInfo.GetChildQueues().Count);
IList <QueueUserACLInfo> listAclInfo = yarnClient.GetQueueAclsInfo();
foreach (QueueUserACLInfo aclInfo in listAclInfo)
{
foreach (QueueACL userAcl in aclInfo.GetUserAcls())
{
Log.Info("User ACL Info for Queue" + ", queueName=" + aclInfo.GetQueueName() + ", userAcl="
+ userAcl.ToString());
}
}
if (domainId != null && domainId.Length > 0 && toCreateDomain)
{
PrepareTimelineDomain();
}
// Get a new application id
YarnClientApplication app = yarnClient.CreateApplication();
GetNewApplicationResponse appResponse = app.GetNewApplicationResponse();
// TODO get min/max resource capabilities from RM and change memory ask if needed
// If we do not have min/max, we may not be able to correctly request
// the required resources from the RM for the app master
// Memory ask has to be a multiple of min and less than max.
// Dump out information about cluster capability as seen by the resource manager
int maxMem = appResponse.GetMaximumResourceCapability().GetMemory();
Log.Info("Max mem capabililty of resources in this cluster " + maxMem);
// A resource ask cannot exceed the max.
if (amMemory > maxMem)
{
Log.Info("AM memory specified above max threshold of cluster. Using max value." +
", specified=" + amMemory + ", max=" + maxMem);
amMemory = maxMem;
}
int maxVCores = appResponse.GetMaximumResourceCapability().GetVirtualCores();
Log.Info("Max virtual cores capabililty of resources in this cluster " + maxVCores
);
if (amVCores > maxVCores)
{
Log.Info("AM virtual cores specified above max threshold of cluster. " + "Using max value."
+ ", specified=" + amVCores + ", max=" + maxVCores);
amVCores = maxVCores;
}
// set the application name
ApplicationSubmissionContext appContext = app.GetApplicationSubmissionContext();
ApplicationId appId = appContext.GetApplicationId();
appContext.SetKeepContainersAcrossApplicationAttempts(keepContainers);
appContext.SetApplicationName(appName);
if (attemptFailuresValidityInterval >= 0)
{
appContext.SetAttemptFailuresValidityInterval(attemptFailuresValidityInterval);
}
// set local resources for the application master
// local files or archives as needed
// In this scenario, the jar file for the application master is part of the local resources
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
Log.Info("Copy App Master jar from local filesystem and add to local environment"
);
// Copy the application master jar to the filesystem
// Create a local resource to point to the destination jar path
FileSystem fs = FileSystem.Get(conf);
AddToLocalResources(fs, appMasterJar, appMasterJarPath, appId.ToString(), localResources
, null);
// Set the log4j properties if needed
if (!log4jPropFile.IsEmpty())
{
AddToLocalResources(fs, log4jPropFile, log4jPath, appId.ToString(), localResources
, null);
}
// The shell script has to be made available on the final container(s)
// where it will be executed.
// To do this, we need to first copy into the filesystem that is visible
// to the yarn framework.
// We do not need to set this as a local resource for the application
// master as the application master does not need it.
string hdfsShellScriptLocation = string.Empty;
long hdfsShellScriptLen = 0;
long hdfsShellScriptTimestamp = 0;
if (!shellScriptPath.IsEmpty())
{
Path shellSrc = new Path(shellScriptPath);
string shellPathSuffix = appName + "/" + appId.ToString() + "/" + ScriptPath;
Path shellDst = new Path(fs.GetHomeDirectory(), shellPathSuffix);
fs.CopyFromLocalFile(false, true, shellSrc, shellDst);
hdfsShellScriptLocation = shellDst.ToUri().ToString();
FileStatus shellFileStatus = fs.GetFileStatus(shellDst);
hdfsShellScriptLen = shellFileStatus.GetLen();
hdfsShellScriptTimestamp = shellFileStatus.GetModificationTime();
}
if (!shellCommand.IsEmpty())
{
AddToLocalResources(fs, null, shellCommandPath, appId.ToString(), localResources,
shellCommand);
}
if (shellArgs.Length > 0)
{
AddToLocalResources(fs, null, shellArgsPath, appId.ToString(), localResources, StringUtils
.Join(shellArgs, " "));
}
// Set the necessary security tokens as needed
//amContainer.setContainerTokens(containerToken);
// Set the env variables to be setup in the env where the application master will be run
Log.Info("Set the environment for the application master");
IDictionary <string, string> env = new Dictionary <string, string>();
// put location of shell script into env
// using the env info, the application master will create the correct local resource for the
// eventual containers that will be launched to execute the shell scripts
env[DSConstants.Distributedshellscriptlocation] = hdfsShellScriptLocation;
env[DSConstants.Distributedshellscripttimestamp] = System.Convert.ToString(hdfsShellScriptTimestamp
);
env[DSConstants.Distributedshellscriptlen] = System.Convert.ToString(hdfsShellScriptLen
);
if (domainId != null && domainId.Length > 0)
{
env[DSConstants.Distributedshelltimelinedomain] = domainId;
}
// Add AppMaster.jar location to classpath
// At some point we should not be required to add
// the hadoop specific classpaths to the env.
// It should be provided out of the box.
// For now setting all required classpaths including
// the classpath to "." for the application jar
StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.Classpath
.$$()).Append(ApplicationConstants.ClassPathSeparator).Append("./*");
foreach (string c in conf.GetStrings(YarnConfiguration.YarnApplicationClasspath,
YarnConfiguration.DefaultYarnCrossPlatformApplicationClasspath))
{
classPathEnv.Append(ApplicationConstants.ClassPathSeparator);
classPathEnv.Append(c.Trim());
}
classPathEnv.Append(ApplicationConstants.ClassPathSeparator).Append("./log4j.properties"
);
// add the runtime classpath needed for tests to work
if (conf.GetBoolean(YarnConfiguration.IsMiniYarnCluster, false))
{
classPathEnv.Append(':');
classPathEnv.Append(Runtime.GetProperty("java.class.path"));
}
env["CLASSPATH"] = classPathEnv.ToString();
// Set the necessary command to execute the application master
Vector <CharSequence> vargs = new Vector <CharSequence>(30);
// Set java executable command
Log.Info("Setting up app master command");
vargs.AddItem(ApplicationConstants.Environment.JavaHome.$$() + "/bin/java");
// Set Xmx based on am memory size
vargs.AddItem("-Xmx" + amMemory + "m");
// Set class name
vargs.AddItem(appMasterMainClass);
// Set params for Application Master
vargs.AddItem("--container_memory " + containerMemory.ToString());
vargs.AddItem("--container_vcores " + containerVirtualCores.ToString());
vargs.AddItem("--num_containers " + numContainers.ToString());
if (null != nodeLabelExpression)
{
appContext.SetNodeLabelExpression(nodeLabelExpression);
}
vargs.AddItem("--priority " + shellCmdPriority.ToString());
foreach (KeyValuePair <string, string> entry in shellEnv)
{
vargs.AddItem("--shell_env " + entry.Key + "=" + entry.Value);
}
if (debugFlag)
{
vargs.AddItem("--debug");
}
vargs.AddItem("1>" + ApplicationConstants.LogDirExpansionVar + "/AppMaster.stdout"
);
vargs.AddItem("2>" + ApplicationConstants.LogDirExpansionVar + "/AppMaster.stderr"
);
// Get final commmand
StringBuilder command = new StringBuilder();
foreach (CharSequence str in vargs)
{
command.Append(str).Append(" ");
}
Log.Info("Completed setting up app master command " + command.ToString());
IList <string> commands = new AList <string>();
commands.AddItem(command.ToString());
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = ContainerLaunchContext.NewInstance(localResources
, env, commands, null, null, null);
// Set up resource type requirements
// For now, both memory and vcores are supported, so we set memory and
// vcores requirements
Resource capability = Resource.NewInstance(amMemory, amVCores);
appContext.SetResource(capability);
// Service data is a binary blob that can be passed to the application
// Not needed in this scenario
// amContainer.setServiceData(serviceData);
// Setup security tokens
if (UserGroupInformation.IsSecurityEnabled())
{
// Note: Credentials class is marked as LimitedPrivate for HDFS and MapReduce
Credentials credentials = new Credentials();
string tokenRenewer = conf.Get(YarnConfiguration.RmPrincipal);
if (tokenRenewer == null || tokenRenewer.Length == 0)
{
throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"
);
}
// For now, only getting tokens for the default file-system.
Org.Apache.Hadoop.Security.Token.Token <object>[] tokens = fs.AddDelegationTokens(
tokenRenewer, credentials);
if (tokens != null)
{
foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in tokens)
{
Log.Info("Got dt for " + fs.GetUri() + "; " + token);
}
}
DataOutputBuffer dob = new DataOutputBuffer();
credentials.WriteTokenStorageToStream(dob);
ByteBuffer fsTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
amContainer.SetTokens(fsTokens);
}
appContext.SetAMContainerSpec(amContainer);
// Set the priority for the application master
// TODO - what is the range for priority? how to decide?
Priority pri = Priority.NewInstance(amPriority);
appContext.SetPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.SetQueue(amQueue);
// Submit the application to the applications manager
// SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest);
// Ignore the response as either a valid response object is returned on success
// or an exception thrown to denote some form of a failure
Log.Info("Submitting application to ASM");
yarnClient.SubmitApplication(appContext);
// TODO
// Try submitting the same request again
// app submission failure?
// Monitor the application
return(MonitorApplication(appId));
}