internal JobQueueInfo(QueueInfo queue)
: this(queue.GetQueueName(), queue.GetSchedulingInfo())
{
SetQueueState(queue.GetState().GetStateName());
SetQueueChildren(queue.GetQueueChildren());
SetProperties(queue.GetProperties());
SetJobStatuses(queue.GetJobStatuses());
}
/// <summary>
/// Utility method to validate a resource request, by insuring that the
/// requested memory/vcore is non-negative and not greater than max
/// </summary>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.InvalidResourceRequestException
/// ">when there is invalid request</exception>
private static void ValidateResourceRequest(ResourceRequest resReq, Org.Apache.Hadoop.Yarn.Api.Records.Resource
maximumResource, QueueInfo queueInfo, RMContext rmContext)
{
if (resReq.GetCapability().GetMemory() < 0 || resReq.GetCapability().GetMemory()
> maximumResource.GetMemory())
{
throw new InvalidResourceRequestException("Invalid resource request" + ", requested memory < 0"
+ ", or requested memory > max configured" + ", requestedMemory=" + resReq.GetCapability
().GetMemory() + ", maxMemory=" + maximumResource.GetMemory());
}
if (resReq.GetCapability().GetVirtualCores() < 0 || resReq.GetCapability().GetVirtualCores
() > maximumResource.GetVirtualCores())
{
throw new InvalidResourceRequestException("Invalid resource request" + ", requested virtual cores < 0"
+ ", or requested virtual cores > max configured" + ", requestedVirtualCores="
+ resReq.GetCapability().GetVirtualCores() + ", maxVirtualCores=" + maximumResource
.GetVirtualCores());
}
string labelExp = resReq.GetNodeLabelExpression();
// we don't allow specify label expression other than resourceName=ANY now
if (!ResourceRequest.Any.Equals(resReq.GetResourceName()) && labelExp != null &&
!labelExp.Trim().IsEmpty())
{
throw new InvalidResourceRequestException("Invailid resource request, queue=" + queueInfo
.GetQueueName() + " specified node label expression in a " + "resource request has resource name = "
+ resReq.GetResourceName());
}
// we don't allow specify label expression with more than one node labels now
if (labelExp != null && labelExp.Contains("&&"))
{
throw new InvalidResourceRequestException("Invailid resource request, queue=" + queueInfo
.GetQueueName() + " specified more than one node label " + "in a node label expression, node label expression = "
+ labelExp);
}
if (labelExp != null && !labelExp.Trim().IsEmpty() && queueInfo != null)
{
if (!CheckQueueLabelExpression(queueInfo.GetAccessibleNodeLabels(), labelExp, rmContext
))
{
throw new InvalidResourceRequestException("Invalid resource request" + ", queue="
+ queueInfo.GetQueueName() + " doesn't have permission to access all labels " +
"in resource request. labelExpression of resource request=" + labelExp + ". Queue labels="
+ (queueInfo.GetAccessibleNodeLabels() == null ? string.Empty : StringUtils.Join
(queueInfo.GetAccessibleNodeLabels().GetEnumerator(), ',')));
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestNormalizeNodeLabelExpression()
{
// mock queue and scheduler
YarnScheduler scheduler = Org.Mockito.Mockito.Mock <YarnScheduler>();
ICollection <string> queueAccessibleNodeLabels = Sets.NewHashSet();
QueueInfo queueInfo = Org.Mockito.Mockito.Mock <QueueInfo>();
Org.Mockito.Mockito.When(queueInfo.GetQueueName()).ThenReturn("queue");
Org.Mockito.Mockito.When(queueInfo.GetAccessibleNodeLabels()).ThenReturn(queueAccessibleNodeLabels
);
Org.Mockito.Mockito.When(queueInfo.GetDefaultNodeLabelExpression()).ThenReturn(" x "
);
Org.Mockito.Mockito.When(scheduler.GetQueueInfo(Matchers.Any <string>(), Matchers.AnyBoolean
(), Matchers.AnyBoolean())).ThenReturn(queueInfo);
Org.Apache.Hadoop.Yarn.Api.Records.Resource maxResource = Resources.CreateResource
(YarnConfiguration.DefaultRmSchedulerMaximumAllocationMb, YarnConfiguration.DefaultRmSchedulerMaximumAllocationVcores
);
// queue has labels, success cases
try
{
// set queue accessible node labels to [x, y]
queueAccessibleNodeLabels.Clear();
Sharpen.Collections.AddAll(queueAccessibleNodeLabels, Arrays.AsList("x", "y"));
rmContext.GetNodeLabelManager().AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), ResourceRequest.Any, resource, 1);
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
NUnit.Framework.Assert.IsTrue(resReq.GetNodeLabelExpression().Equals("x"));
resReq.SetNodeLabelExpression(" y ");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
NUnit.Framework.Assert.IsTrue(resReq.GetNodeLabelExpression().Equals("y"));
}
catch (InvalidResourceRequestException e)
{
Sharpen.Runtime.PrintStackTrace(e);
NUnit.Framework.Assert.Fail("Should be valid when request labels is a subset of queue labels"
);
}
finally
{
rmContext.GetNodeLabelManager().RemoveFromClusterNodeLabels(Arrays.AsList("x", "y"
));
}
}
private void PrintQueueInfo(PrintWriter writer, QueueInfo queueInfo)
{
writer.Write("Queue Name : ");
writer.WriteLine(queueInfo.GetQueueName());
writer.Write("\tState : ");
writer.WriteLine(queueInfo.GetQueueState());
DecimalFormat df = new DecimalFormat("#.0");
writer.Write("\tCapacity : ");
writer.WriteLine(df.Format(queueInfo.GetCapacity() * 100) + "%");
writer.Write("\tCurrent Capacity : ");
writer.WriteLine(df.Format(queueInfo.GetCurrentCapacity() * 100) + "%");
writer.Write("\tMaximum Capacity : ");
writer.WriteLine(df.Format(queueInfo.GetMaximumCapacity() * 100) + "%");
writer.Write("\tDefault Node Label expression : ");
if (null != queueInfo.GetDefaultNodeLabelExpression())
{
writer.WriteLine(queueInfo.GetDefaultNodeLabelExpression());
}
else
{
writer.WriteLine();
}
ICollection <string> nodeLabels = queueInfo.GetAccessibleNodeLabels();
StringBuilder labelList = new StringBuilder();
writer.Write("\tAccessible Node Labels : ");
foreach (string nodeLabel in nodeLabels)
{
if (labelList.Length > 0)
{
labelList.Append(',');
}
labelList.Append(nodeLabel);
}
writer.WriteLine(labelList.ToString());
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestValidateResourceRequestWithErrorLabelsPermission()
{
// mock queue and scheduler
YarnScheduler scheduler = Org.Mockito.Mockito.Mock <YarnScheduler>();
ICollection <string> queueAccessibleNodeLabels = Sets.NewHashSet();
QueueInfo queueInfo = Org.Mockito.Mockito.Mock <QueueInfo>();
Org.Mockito.Mockito.When(queueInfo.GetQueueName()).ThenReturn("queue");
Org.Mockito.Mockito.When(queueInfo.GetAccessibleNodeLabels()).ThenReturn(queueAccessibleNodeLabels
);
Org.Mockito.Mockito.When(scheduler.GetQueueInfo(Matchers.Any <string>(), Matchers.AnyBoolean
(), Matchers.AnyBoolean())).ThenReturn(queueInfo);
Org.Apache.Hadoop.Yarn.Api.Records.Resource maxResource = Resources.CreateResource
(YarnConfiguration.DefaultRmSchedulerMaximumAllocationMb, YarnConfiguration.DefaultRmSchedulerMaximumAllocationVcores
);
// queue has labels, success cases
try
{
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.Clear();
Sharpen.Collections.AddAll(queueAccessibleNodeLabels, Arrays.AsList("x", "y"));
rmContext.GetNodeLabelManager().AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), ResourceRequest.Any, resource, 1);
resReq.SetNodeLabelExpression("x");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
resReq.SetNodeLabelExpression("y");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
resReq.SetNodeLabelExpression(string.Empty);
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
resReq.SetNodeLabelExpression(" ");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
}
catch (InvalidResourceRequestException e)
{
Sharpen.Runtime.PrintStackTrace(e);
NUnit.Framework.Assert.Fail("Should be valid when request labels is a subset of queue labels"
);
}
finally
{
rmContext.GetNodeLabelManager().RemoveFromClusterNodeLabels(Arrays.AsList("x", "y"
));
}
// same as above, but cluster node labels don't contains label being
// requested. should fail
try
{
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.Clear();
Sharpen.Collections.AddAll(queueAccessibleNodeLabels, Arrays.AsList("x", "y"));
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), ResourceRequest.Any, resource, 1);
resReq.SetNodeLabelExpression("x");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
NUnit.Framework.Assert.Fail("Should fail");
}
catch (InvalidResourceRequestException)
{
}
// queue has labels, failed cases (when ask a label not included by queue)
try
{
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.Clear();
Sharpen.Collections.AddAll(queueAccessibleNodeLabels, Arrays.AsList("x", "y"));
rmContext.GetNodeLabelManager().AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), ResourceRequest.Any, resource, 1);
resReq.SetNodeLabelExpression("z");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
NUnit.Framework.Assert.Fail("Should fail");
}
catch (InvalidResourceRequestException)
{
}
finally
{
rmContext.GetNodeLabelManager().RemoveFromClusterNodeLabels(Arrays.AsList("x", "y"
));
}
// we don't allow specify more than two node labels in a single expression
// now
try
{
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.Clear();
Sharpen.Collections.AddAll(queueAccessibleNodeLabels, Arrays.AsList("x", "y"));
rmContext.GetNodeLabelManager().AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), ResourceRequest.Any, resource, 1);
resReq.SetNodeLabelExpression("x && y");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
NUnit.Framework.Assert.Fail("Should fail");
}
catch (InvalidResourceRequestException)
{
}
finally
{
rmContext.GetNodeLabelManager().RemoveFromClusterNodeLabels(Arrays.AsList("x", "y"
));
}
// queue doesn't have label, succeed (when request no label)
queueAccessibleNodeLabels.Clear();
try
{
// set queue accessible node labels to empty
queueAccessibleNodeLabels.Clear();
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), ResourceRequest.Any, resource, 1);
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
resReq.SetNodeLabelExpression(string.Empty);
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
resReq.SetNodeLabelExpression(" ");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
}
catch (InvalidResourceRequestException e)
{
Sharpen.Runtime.PrintStackTrace(e);
NUnit.Framework.Assert.Fail("Should be valid when request labels is empty");
}
// queue doesn't have label, failed (when request any label)
try
{
// set queue accessible node labels to empty
queueAccessibleNodeLabels.Clear();
rmContext.GetNodeLabelManager().AddToCluserNodeLabels(ImmutableSet.Of("x"));
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), ResourceRequest.Any, resource, 1);
resReq.SetNodeLabelExpression("x");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
NUnit.Framework.Assert.Fail("Should fail");
}
catch (InvalidResourceRequestException)
{
}
finally
{
rmContext.GetNodeLabelManager().RemoveFromClusterNodeLabels(Arrays.AsList("x"));
}
// queue is "*", always succeeded
try
{
// set queue accessible node labels to empty
queueAccessibleNodeLabels.Clear();
queueAccessibleNodeLabels.AddItem(RMNodeLabelsManager.Any);
rmContext.GetNodeLabelManager().AddToCluserNodeLabels(ImmutableSet.Of("x", "y", "z"
));
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), ResourceRequest.Any, resource, 1);
resReq.SetNodeLabelExpression("x");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
resReq.SetNodeLabelExpression("y");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
resReq.SetNodeLabelExpression("z");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
}
catch (InvalidResourceRequestException e)
{
Sharpen.Runtime.PrintStackTrace(e);
NUnit.Framework.Assert.Fail("Should be valid when queue can access any labels");
}
finally
{
rmContext.GetNodeLabelManager().RemoveFromClusterNodeLabels(Arrays.AsList("x", "y"
, "z"));
}
// same as above, but cluster node labels don't contains label, should fail
try
{
// set queue accessible node labels to empty
queueAccessibleNodeLabels.Clear();
queueAccessibleNodeLabels.AddItem(RMNodeLabelsManager.Any);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), ResourceRequest.Any, resource, 1);
resReq.SetNodeLabelExpression("x");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
NUnit.Framework.Assert.Fail("Should fail");
}
catch (InvalidResourceRequestException)
{
}
// we don't allow resource name other than ANY and specify label
try
{
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.Clear();
Sharpen.Collections.AddAll(queueAccessibleNodeLabels, Arrays.AsList("x", "y"));
rmContext.GetNodeLabelManager().AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), "rack", resource, 1);
resReq.SetNodeLabelExpression("x");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
NUnit.Framework.Assert.Fail("Should fail");
}
catch (InvalidResourceRequestException)
{
}
finally
{
rmContext.GetNodeLabelManager().RemoveFromClusterNodeLabels(Arrays.AsList("x", "y"
));
}
// we don't allow resource name other than ANY and specify label even if
// queue has accessible label = *
try
{
// set queue accessible node labesl to *
queueAccessibleNodeLabels.Clear();
Sharpen.Collections.AddAll(queueAccessibleNodeLabels, Arrays.AsList(CommonNodeLabelsManager
.Any));
rmContext.GetNodeLabelManager().AddToCluserNodeLabels(ImmutableSet.Of("x"));
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = Resources.CreateResource(0
, YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores);
ResourceRequest resReq = BuilderUtils.NewResourceRequest(Org.Mockito.Mockito.Mock
<Priority>(), "rack", resource, 1);
resReq.SetNodeLabelExpression("x");
SchedulerUtils.NormalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler
, rmContext);
NUnit.Framework.Assert.Fail("Should fail");
}
catch (InvalidResourceRequestException)
{
}
finally
{
rmContext.GetNodeLabelManager().RemoveFromClusterNodeLabels(Arrays.AsList("x"));
}
}
/// <summary>Main run function for the client</summary>
/// <returns>true if application completed successfully</returns>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual bool Run()
{
Log.Info("Running Client");
yarnClient.Start();
YarnClusterMetrics clusterMetrics = yarnClient.GetYarnClusterMetrics();
Log.Info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics
.GetNumNodeManagers());
IList <NodeReport> clusterNodeReports = yarnClient.GetNodeReports(NodeState.Running
);
Log.Info("Got Cluster node info from ASM");
foreach (NodeReport node in clusterNodeReports)
{
Log.Info("Got node report from ASM for" + ", nodeId=" + node.GetNodeId() + ", nodeAddress"
+ node.GetHttpAddress() + ", nodeRackName" + node.GetRackName() + ", nodeNumContainers"
+ node.GetNumContainers());
}
QueueInfo queueInfo = yarnClient.GetQueueInfo(this.amQueue);
Log.Info("Queue info" + ", queueName=" + queueInfo.GetQueueName() + ", queueCurrentCapacity="
+ queueInfo.GetCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.GetMaximumCapacity
() + ", queueApplicationCount=" + queueInfo.GetApplications().Count + ", queueChildQueueCount="
+ queueInfo.GetChildQueues().Count);
IList <QueueUserACLInfo> listAclInfo = yarnClient.GetQueueAclsInfo();
foreach (QueueUserACLInfo aclInfo in listAclInfo)
{
foreach (QueueACL userAcl in aclInfo.GetUserAcls())
{
Log.Info("User ACL Info for Queue" + ", queueName=" + aclInfo.GetQueueName() + ", userAcl="
+ userAcl.ToString());
}
}
if (domainId != null && domainId.Length > 0 && toCreateDomain)
{
PrepareTimelineDomain();
}
// Get a new application id
YarnClientApplication app = yarnClient.CreateApplication();
GetNewApplicationResponse appResponse = app.GetNewApplicationResponse();
// TODO get min/max resource capabilities from RM and change memory ask if needed
// If we do not have min/max, we may not be able to correctly request
// the required resources from the RM for the app master
// Memory ask has to be a multiple of min and less than max.
// Dump out information about cluster capability as seen by the resource manager
int maxMem = appResponse.GetMaximumResourceCapability().GetMemory();
Log.Info("Max mem capabililty of resources in this cluster " + maxMem);
// A resource ask cannot exceed the max.
if (amMemory > maxMem)
{
Log.Info("AM memory specified above max threshold of cluster. Using max value." +
", specified=" + amMemory + ", max=" + maxMem);
amMemory = maxMem;
}
int maxVCores = appResponse.GetMaximumResourceCapability().GetVirtualCores();
Log.Info("Max virtual cores capabililty of resources in this cluster " + maxVCores
);
if (amVCores > maxVCores)
{
Log.Info("AM virtual cores specified above max threshold of cluster. " + "Using max value."
+ ", specified=" + amVCores + ", max=" + maxVCores);
amVCores = maxVCores;
}
// set the application name
ApplicationSubmissionContext appContext = app.GetApplicationSubmissionContext();
ApplicationId appId = appContext.GetApplicationId();
appContext.SetKeepContainersAcrossApplicationAttempts(keepContainers);
appContext.SetApplicationName(appName);
if (attemptFailuresValidityInterval >= 0)
{
appContext.SetAttemptFailuresValidityInterval(attemptFailuresValidityInterval);
}
// set local resources for the application master
// local files or archives as needed
// In this scenario, the jar file for the application master is part of the local resources
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
Log.Info("Copy App Master jar from local filesystem and add to local environment"
);
// Copy the application master jar to the filesystem
// Create a local resource to point to the destination jar path
FileSystem fs = FileSystem.Get(conf);
AddToLocalResources(fs, appMasterJar, appMasterJarPath, appId.ToString(), localResources
, null);
// Set the log4j properties if needed
if (!log4jPropFile.IsEmpty())
{
AddToLocalResources(fs, log4jPropFile, log4jPath, appId.ToString(), localResources
, null);
}
// The shell script has to be made available on the final container(s)
// where it will be executed.
// To do this, we need to first copy into the filesystem that is visible
// to the yarn framework.
// We do not need to set this as a local resource for the application
// master as the application master does not need it.
string hdfsShellScriptLocation = string.Empty;
long hdfsShellScriptLen = 0;
long hdfsShellScriptTimestamp = 0;
if (!shellScriptPath.IsEmpty())
{
Path shellSrc = new Path(shellScriptPath);
string shellPathSuffix = appName + "/" + appId.ToString() + "/" + ScriptPath;
Path shellDst = new Path(fs.GetHomeDirectory(), shellPathSuffix);
fs.CopyFromLocalFile(false, true, shellSrc, shellDst);
hdfsShellScriptLocation = shellDst.ToUri().ToString();
FileStatus shellFileStatus = fs.GetFileStatus(shellDst);
hdfsShellScriptLen = shellFileStatus.GetLen();
hdfsShellScriptTimestamp = shellFileStatus.GetModificationTime();
}
if (!shellCommand.IsEmpty())
{
AddToLocalResources(fs, null, shellCommandPath, appId.ToString(), localResources,
shellCommand);
}
if (shellArgs.Length > 0)
{
AddToLocalResources(fs, null, shellArgsPath, appId.ToString(), localResources, StringUtils
.Join(shellArgs, " "));
}
// Set the necessary security tokens as needed
//amContainer.setContainerTokens(containerToken);
// Set the env variables to be setup in the env where the application master will be run
Log.Info("Set the environment for the application master");
IDictionary <string, string> env = new Dictionary <string, string>();
// put location of shell script into env
// using the env info, the application master will create the correct local resource for the
// eventual containers that will be launched to execute the shell scripts
env[DSConstants.Distributedshellscriptlocation] = hdfsShellScriptLocation;
env[DSConstants.Distributedshellscripttimestamp] = System.Convert.ToString(hdfsShellScriptTimestamp
);
env[DSConstants.Distributedshellscriptlen] = System.Convert.ToString(hdfsShellScriptLen
);
if (domainId != null && domainId.Length > 0)
{
env[DSConstants.Distributedshelltimelinedomain] = domainId;
}
// Add AppMaster.jar location to classpath
// At some point we should not be required to add
// the hadoop specific classpaths to the env.
// It should be provided out of the box.
// For now setting all required classpaths including
// the classpath to "." for the application jar
StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.Classpath
.$$()).Append(ApplicationConstants.ClassPathSeparator).Append("./*");
foreach (string c in conf.GetStrings(YarnConfiguration.YarnApplicationClasspath,
YarnConfiguration.DefaultYarnCrossPlatformApplicationClasspath))
{
classPathEnv.Append(ApplicationConstants.ClassPathSeparator);
classPathEnv.Append(c.Trim());
}
classPathEnv.Append(ApplicationConstants.ClassPathSeparator).Append("./log4j.properties"
);
// add the runtime classpath needed for tests to work
if (conf.GetBoolean(YarnConfiguration.IsMiniYarnCluster, false))
{
classPathEnv.Append(':');
classPathEnv.Append(Runtime.GetProperty("java.class.path"));
}
env["CLASSPATH"] = classPathEnv.ToString();
// Set the necessary command to execute the application master
Vector <CharSequence> vargs = new Vector <CharSequence>(30);
// Set java executable command
Log.Info("Setting up app master command");
vargs.AddItem(ApplicationConstants.Environment.JavaHome.$$() + "/bin/java");
// Set Xmx based on am memory size
vargs.AddItem("-Xmx" + amMemory + "m");
// Set class name
vargs.AddItem(appMasterMainClass);
// Set params for Application Master
vargs.AddItem("--container_memory " + containerMemory.ToString());
vargs.AddItem("--container_vcores " + containerVirtualCores.ToString());
vargs.AddItem("--num_containers " + numContainers.ToString());
if (null != nodeLabelExpression)
{
appContext.SetNodeLabelExpression(nodeLabelExpression);
}
vargs.AddItem("--priority " + shellCmdPriority.ToString());
foreach (KeyValuePair <string, string> entry in shellEnv)
{
vargs.AddItem("--shell_env " + entry.Key + "=" + entry.Value);
}
if (debugFlag)
{
vargs.AddItem("--debug");
}
vargs.AddItem("1>" + ApplicationConstants.LogDirExpansionVar + "/AppMaster.stdout"
);
vargs.AddItem("2>" + ApplicationConstants.LogDirExpansionVar + "/AppMaster.stderr"
);
// Get final commmand
StringBuilder command = new StringBuilder();
foreach (CharSequence str in vargs)
{
command.Append(str).Append(" ");
}
Log.Info("Completed setting up app master command " + command.ToString());
IList <string> commands = new AList <string>();
commands.AddItem(command.ToString());
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = ContainerLaunchContext.NewInstance(localResources
, env, commands, null, null, null);
// Set up resource type requirements
// For now, both memory and vcores are supported, so we set memory and
// vcores requirements
Resource capability = Resource.NewInstance(amMemory, amVCores);
appContext.SetResource(capability);
// Service data is a binary blob that can be passed to the application
// Not needed in this scenario
// amContainer.setServiceData(serviceData);
// Setup security tokens
if (UserGroupInformation.IsSecurityEnabled())
{
// Note: Credentials class is marked as LimitedPrivate for HDFS and MapReduce
Credentials credentials = new Credentials();
string tokenRenewer = conf.Get(YarnConfiguration.RmPrincipal);
if (tokenRenewer == null || tokenRenewer.Length == 0)
{
throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"
);
}
// For now, only getting tokens for the default file-system.
Org.Apache.Hadoop.Security.Token.Token <object>[] tokens = fs.AddDelegationTokens(
tokenRenewer, credentials);
if (tokens != null)
{
foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in tokens)
{
Log.Info("Got dt for " + fs.GetUri() + "; " + token);
}
}
DataOutputBuffer dob = new DataOutputBuffer();
credentials.WriteTokenStorageToStream(dob);
ByteBuffer fsTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
amContainer.SetTokens(fsTokens);
}
appContext.SetAMContainerSpec(amContainer);
// Set the priority for the application master
// TODO - what is the range for priority? how to decide?
Priority pri = Priority.NewInstance(amPriority);
appContext.SetPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.SetQueue(amQueue);
// Submit the application to the applications manager
// SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest);
// Ignore the response as either a valid response object is returned on success
// or an exception thrown to denote some form of a failure
Log.Info("Submitting application to ASM");
yarnClient.SubmitApplication(appContext);
// TODO
// Try submitting the same request again
// app submission failure?
// Monitor the application
return(MonitorApplication(appId));
}
