/// <exception cref="System.Exception"/>
private ApplicationId SubmitAppAndGetAppId(AccessControlList viewACL, AccessControlList
modifyACL)
{
SubmitApplicationRequest submitRequest = recordFactory.NewRecordInstance <SubmitApplicationRequest
>();
ApplicationSubmissionContext context = recordFactory.NewRecordInstance <ApplicationSubmissionContext
>();
ApplicationId applicationId = rmClient.GetNewApplication(recordFactory.NewRecordInstance
<GetNewApplicationRequest>()).GetApplicationId();
context.SetApplicationId(applicationId);
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>();
acls[ApplicationAccessType.ViewApp] = viewACL.GetAclString();
acls[ApplicationAccessType.ModifyApp] = modifyACL.GetAclString();
ContainerLaunchContext amContainer = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
Resource resource = BuilderUtils.NewResource(1024, 1);
context.SetResource(resource);
amContainer.SetApplicationACLs(acls);
context.SetAMContainerSpec(amContainer);
submitRequest.SetApplicationSubmissionContext(context);
rmClient.SubmitApplication(submitRequest);
resourceManager.WaitForState(applicationId, RMAppState.Accepted);
return(applicationId);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override JobStatus SubmitJob(JobID jobId, string jobSubmitDir, Credentials
ts)
{
AddHistoryToken(ts);
// Construct necessary information to start the MR AM
ApplicationSubmissionContext appContext = CreateApplicationSubmissionContext(conf
, jobSubmitDir, ts);
// Submit to ResourceManager
try
{
ApplicationId applicationId = resMgrDelegate.SubmitApplication(appContext);
ApplicationReport appMaster = resMgrDelegate.GetApplicationReport(applicationId);
string diagnostics = (appMaster == null ? "application report is null" : appMaster
.GetDiagnostics());
if (appMaster == null || appMaster.GetYarnApplicationState() == YarnApplicationState
.Failed || appMaster.GetYarnApplicationState() == YarnApplicationState.Killed)
{
throw new IOException("Failed to run job : " + diagnostics);
}
return(clientCache.GetClient(jobId).GetJobStatus(jobId));
}
catch (YarnException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual void Submit()
{
lock (this)
{
ApplicationSubmissionContext context = recordFactory.NewRecordInstance <ApplicationSubmissionContext
>();
context.SetApplicationId(this.applicationId);
context.SetQueue(this.queue);
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ContainerLaunchContext>();
context.SetAMContainerSpec(amContainer);
context.SetResource(Resources.CreateResource(YarnConfiguration.DefaultRmSchedulerMinimumAllocationMb
));
SubmitApplicationRequest request = recordFactory.NewRecordInstance <SubmitApplicationRequest
>();
request.SetApplicationSubmissionContext(context);
ResourceScheduler scheduler = resourceManager.GetResourceScheduler();
resourceManager.GetClientRMService().SubmitApplication(request);
// Notify scheduler
AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(this.applicationId
, this.queue, "user");
scheduler.Handle(addAppEvent);
AppAttemptAddedSchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent
(this.applicationAttemptId, false);
scheduler.Handle(addAttemptEvent);
}
}
/// <exception cref="System.Exception"/>
private ApplicationId SubmitAppAndGetAppId(string submitter, string queueName, bool
setupACLs)
{
GetNewApplicationRequest newAppRequest = GetNewApplicationRequest.NewInstance();
ApplicationClientProtocol submitterClient = GetRMClientForUser(submitter);
ApplicationId applicationId = submitterClient.GetNewApplication(newAppRequest).GetApplicationId
();
Resource resource = BuilderUtils.NewResource(1024, 1);
IDictionary <ApplicationAccessType, string> acls = CreateACLs(submitter, setupACLs
);
ContainerLaunchContext amContainerSpec = ContainerLaunchContext.NewInstance(null,
null, null, null, null, acls);
ApplicationSubmissionContext appSubmissionContext = ApplicationSubmissionContext.
NewInstance(applicationId, "applicationName", queueName, null, amContainerSpec,
false, true, 1, resource, "applicationType");
appSubmissionContext.SetApplicationId(applicationId);
appSubmissionContext.SetQueue(queueName);
SubmitApplicationRequest submitRequest = SubmitApplicationRequest.NewInstance(appSubmissionContext
);
submitterClient.SubmitApplication(submitRequest);
resourceManager.WaitForState(applicationId, RMAppState.Accepted);
return(applicationId);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private RMAppImpl CreateAndPopulateNewRMApp(ApplicationSubmissionContext submissionContext
, long submitTime, string user, bool isRecovery)
{
ApplicationId applicationId = submissionContext.GetApplicationId();
ResourceRequest amReq = ValidateAndCreateResourceRequest(submissionContext, isRecovery
);
// Create RMApp
RMAppImpl application = new RMAppImpl(applicationId, rmContext, this.conf, submissionContext
.GetApplicationName(), user, submissionContext.GetQueue(), submissionContext, this
.scheduler, this.masterService, submitTime, submissionContext.GetApplicationType
(), submissionContext.GetApplicationTags(), amReq);
// Concurrent app submissions with same applicationId will fail here
// Concurrent app submissions with different applicationIds will not
// influence each other
if (rmContext.GetRMApps().PutIfAbsent(applicationId, application) != null)
{
string message = "Application with id " + applicationId + " is already present! Cannot add a duplicate!";
Log.Warn(message);
throw new YarnException(message);
}
// Inform the ACLs Manager
this.applicationACLsManager.AddApplication(applicationId, submissionContext.GetAMContainerSpec
().GetApplicationACLs());
string appViewACLs = submissionContext.GetAMContainerSpec().GetApplicationACLs()[
ApplicationAccessType.ViewApp];
rmContext.GetSystemMetricsPublisher().AppACLsUpdated(application, appViewACLs, Runtime
.CurrentTimeMillis());
return(application);
}
/// <exception cref="System.Exception"/>
public virtual void TestWarnCommandOpts()
{
Logger logger = Logger.GetLogger(typeof(YARNRunner));
ByteArrayOutputStream bout = new ByteArrayOutputStream();
Layout layout = new SimpleLayout();
Appender appender = new WriterAppender(layout, bout);
logger.AddAppender(appender);
JobConf jobConf = new JobConf();
jobConf.Set(MRJobConfig.MrAmAdminCommandOpts, "-Djava.net.preferIPv4Stack=true -Djava.library.path=foo"
);
jobConf.Set(MRJobConfig.MrAmCommandOpts, "-Xmx1024m -Djava.library.path=bar");
YARNRunner yarnRunner = new YARNRunner(jobConf);
ApplicationSubmissionContext submissionContext = BuildSubmitContext(yarnRunner, jobConf
);
string logMsg = bout.ToString();
NUnit.Framework.Assert.IsTrue(logMsg.Contains("WARN - Usage of -Djava.library.path in "
+ "yarn.app.mapreduce.am.admin-command-opts can cause programs to no " + "longer function if hadoop native libraries are used. These values "
+ "should be set as part of the LD_LIBRARY_PATH in the app master JVM " + "env using yarn.app.mapreduce.am.admin.user.env config settings."
));
NUnit.Framework.Assert.IsTrue(logMsg.Contains("WARN - Usage of -Djava.library.path in "
+ "yarn.app.mapreduce.am.command-opts can cause programs to no longer " + "function if hadoop native libraries are used. These values should "
+ "be set as part of the LD_LIBRARY_PATH in the app master JVM env " + "using yarn.app.mapreduce.am.env config settings."
));
}
public virtual void TestAMStandardEnv()
{
string AdminLibPath = "foo";
string UserLibPath = "bar";
string UserShell = "shell";
JobConf jobConf = new JobConf();
jobConf.Set(MRJobConfig.MrAmAdminUserEnv, "LD_LIBRARY_PATH=" + AdminLibPath);
jobConf.Set(MRJobConfig.MrAmEnv, "LD_LIBRARY_PATH=" + UserLibPath);
jobConf.Set(MRJobConfig.MapredAdminUserShell, UserShell);
YARNRunner yarnRunner = new YARNRunner(jobConf);
ApplicationSubmissionContext appSubCtx = BuildSubmitContext(yarnRunner, jobConf);
// make sure PWD is first in the lib path
ContainerLaunchContext clc = appSubCtx.GetAMContainerSpec();
IDictionary <string, string> env = clc.GetEnvironment();
string libPath = env[ApplicationConstants.Environment.LdLibraryPath.ToString()];
NUnit.Framework.Assert.IsNotNull("LD_LIBRARY_PATH not set", libPath);
string cps = jobConf.GetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, MRConfig
.DefaultMapreduceAppSubmissionCrossPlatform) ? ApplicationConstants.ClassPathSeparator
: FilePath.pathSeparator;
NUnit.Framework.Assert.AreEqual("Bad AM LD_LIBRARY_PATH setting", MRApps.CrossPlatformifyMREnv
(conf, ApplicationConstants.Environment.Pwd) + cps + AdminLibPath + cps + UserLibPath
, libPath);
// make sure SHELL is set
string shell = env[ApplicationConstants.Environment.Shell.ToString()];
NUnit.Framework.Assert.IsNotNull("SHELL not set", shell);
NUnit.Framework.Assert.AreEqual("Bad SHELL setting", UserShell, shell);
}
public SchedulerApplicationAttempt(ApplicationAttemptId applicationAttemptId, string
user, Queue queue, ActiveUsersManager activeUsersManager, RMContext rmContext)
{
// This pendingRelease is used in work-preserving recovery scenario to keep
// track of the AM's outstanding release requests. RM on recovery could
// receive the release request form AM before it receives the container status
// from NM for recovery. In this case, the to-be-recovered containers reported
// by NM should not be recovered.
// Time of the last container scheduled at the current allowed level
Preconditions.CheckNotNull(rmContext, "RMContext should not be null");
this.rmContext = rmContext;
this.appSchedulingInfo = new AppSchedulingInfo(applicationAttemptId, user, queue,
activeUsersManager, rmContext.GetEpoch());
this.queue = queue;
this.pendingRelease = new HashSet <ContainerId>();
this.attemptId = applicationAttemptId;
if (rmContext.GetRMApps() != null && rmContext.GetRMApps().Contains(applicationAttemptId
.GetApplicationId()))
{
ApplicationSubmissionContext appSubmissionContext = rmContext.GetRMApps()[applicationAttemptId
.GetApplicationId()].GetApplicationSubmissionContext();
if (appSubmissionContext != null)
{
unmanagedAM = appSubmissionContext.GetUnmanagedAM();
this.logAggregationContext = appSubmissionContext.GetLogAggregationContext();
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private void Launch()
{
Connect();
ContainerId masterContainerID = masterContainer.GetId();
ApplicationSubmissionContext applicationContext = application.GetSubmissionContext
();
Log.Info("Setting up container " + masterContainer + " for AM " + application.GetAppAttemptId
());
ContainerLaunchContext launchContext = CreateAMContainerLaunchContext(applicationContext
, masterContainerID);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(launchContext
, masterContainer.GetContainerToken());
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
StartContainersResponse response = containerMgrProxy.StartContainers(allRequests);
if (response.GetFailedRequests() != null && response.GetFailedRequests().Contains
(masterContainerID))
{
Exception t = response.GetFailedRequests()[masterContainerID].DeSerialize();
ParseAndThrowException(t);
}
else
{
Log.Info("Done launching container " + masterContainer + " for AM " + application
.GetAppAttemptId());
}
}
/// <exception cref="System.Exception"/>
public virtual void TestAMAdminCommandOpts()
{
JobConf jobConf = new JobConf();
jobConf.Set(MRJobConfig.MrAmAdminCommandOpts, "-Djava.net.preferIPv4Stack=true");
jobConf.Set(MRJobConfig.MrAmCommandOpts, "-Xmx1024m");
YARNRunner yarnRunner = new YARNRunner(jobConf);
ApplicationSubmissionContext submissionContext = BuildSubmitContext(yarnRunner, jobConf
);
ContainerLaunchContext containerSpec = submissionContext.GetAMContainerSpec();
IList <string> commands = containerSpec.GetCommands();
int index = 0;
int adminIndex = 0;
int adminPos = -1;
int userIndex = 0;
int userPos = -1;
int tmpDirPos = -1;
foreach (string command in commands)
{
if (command != null)
{
NUnit.Framework.Assert.IsFalse("Profiler should be disabled by default", command.
Contains(ProfileParams));
adminPos = command.IndexOf("-Djava.net.preferIPv4Stack=true");
if (adminPos >= 0)
{
adminIndex = index;
}
userPos = command.IndexOf("-Xmx1024m");
if (userPos >= 0)
{
userIndex = index;
}
tmpDirPos = command.IndexOf("-Djava.io.tmpdir=");
}
index++;
}
// Check java.io.tmpdir opts are set in the commands
NUnit.Framework.Assert.IsTrue("java.io.tmpdir is not set for AM", tmpDirPos > 0);
// Check both admin java opts and user java opts are in the commands
NUnit.Framework.Assert.IsTrue("AM admin command opts not in the commands.", adminPos
> 0);
NUnit.Framework.Assert.IsTrue("AM user command opts not in the commands.", userPos
> 0);
// Check the admin java opts is before user java opts in the commands
if (adminIndex == userIndex)
{
NUnit.Framework.Assert.IsTrue("AM admin command opts is after user command opts."
, adminPos < userPos);
}
else
{
NUnit.Framework.Assert.IsTrue("AM admin command opts is after user command opts."
, adminIndex < userIndex);
}
}
public static SubmitApplicationRequest NewInstance(ApplicationSubmissionContext context
)
{
SubmitApplicationRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <
SubmitApplicationRequest>();
request.SetApplicationSubmissionContext(context);
return(request);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public override YarnClientApplication CreateApplication()
{
ApplicationSubmissionContext context = Records.NewRecord <ApplicationSubmissionContext
>();
GetNewApplicationResponse newApp = GetNewApplication();
ApplicationId appId = newApp.GetApplicationId();
context.SetApplicationId(appId);
return(new YarnClientApplication(newApp, context));
}
public override void SetApplicationSubmissionContext(ApplicationSubmissionContext
context)
{
MaybeInitBuilder();
if (context == null)
{
builder.ClearApplicationSubmissionContext();
}
this.applicationSubmissionContext = context;
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
protected internal override void SubmitApplication(ApplicationSubmissionContext submissionContext
, long submitTime, string user)
{
//Do nothing, just add the application to RMContext
RMAppImpl application = new RMAppImpl(submissionContext.GetApplicationId(), this.
rmContext, this.conf, submissionContext.GetApplicationName(), user, submissionContext
.GetQueue(), submissionContext, this.rmContext.GetScheduler(), this.rmContext.GetApplicationMasterService
(), submitTime, submissionContext.GetApplicationType(), submissionContext.GetApplicationTags
(), null);
this.rmContext.GetRMApps()[submissionContext.GetApplicationId()] = application;
}
/// <exception cref="System.Exception"/>
protected internal virtual void RecoverApplication(ApplicationStateData appState,
RMStateStore.RMState rmState)
{
ApplicationSubmissionContext appContext = appState.GetApplicationSubmissionContext
();
ApplicationId appId = appContext.GetApplicationId();
// create and recover app.
RMAppImpl application = CreateAndPopulateNewRMApp(appContext, appState.GetSubmitTime
(), appState.GetUser(), true);
application.Handle(new RMAppRecoverEvent(appId, rmState));
}
// TODO: Don't create again and again.
/// <exception cref="System.IO.IOException"/>
private ContainerLaunchContext CreateAMContainerLaunchContext(ApplicationSubmissionContext
applicationMasterContext, ContainerId containerID)
{
// Construct the actual Container
ContainerLaunchContext container = applicationMasterContext.GetAMContainerSpec();
Log.Info("Command to launch container " + containerID + " : " + StringUtils.ArrayToString
(Sharpen.Collections.ToArray(container.GetCommands(), new string[0])));
// Finalize the container
SetupTokens(container, containerID);
return(container);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual JobID GetNewJobID()
{
try
{
this.application = client.CreateApplication().GetApplicationSubmissionContext();
this.applicationId = this.application.GetApplicationId();
return(TypeConverter.FromYarn(applicationId));
}
catch (YarnException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual Credentials ParseCredentials(ApplicationSubmissionContext
application)
{
Credentials credentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
ByteBuffer tokens = application.GetAMContainerSpec().GetTokens();
if (tokens != null)
{
dibb.Reset(tokens);
credentials.ReadTokenStorageStream(dibb);
tokens.Rewind();
}
return(credentials);
}
public override ApplicationSubmissionContext GetApplicationSubmissionContext()
{
YarnServiceProtos.SubmitApplicationRequestProtoOrBuilder p = viaProto ? proto : builder;
if (this.applicationSubmissionContext != null)
{
return(this.applicationSubmissionContext);
}
if (!p.HasApplicationSubmissionContext())
{
return(null);
}
this.applicationSubmissionContext = ConvertFromProtoFormat(p.GetApplicationSubmissionContext
());
return(this.applicationSubmissionContext);
}
public override ApplicationSubmissionContext GetApplicationSubmissionContext()
{
YarnServerResourceManagerRecoveryProtos.ApplicationStateDataProtoOrBuilder p = viaProto
? proto : builder;
if (applicationSubmissionContext != null)
{
return(applicationSubmissionContext);
}
if (!p.HasApplicationSubmissionContext())
{
return(null);
}
applicationSubmissionContext = new ApplicationSubmissionContextPBImpl(p.GetApplicationSubmissionContext
());
return(applicationSubmissionContext);
}
public static ApplicationStateData NewInstance(long submitTime, long startTime, string
user, ApplicationSubmissionContext submissionContext, RMAppState state, string
diagnostics, long finishTime)
{
ApplicationStateData appState = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ApplicationStateData
>();
appState.SetSubmitTime(submitTime);
appState.SetStartTime(startTime);
appState.SetUser(user);
appState.SetApplicationSubmissionContext(submissionContext);
appState.SetState(state);
appState.SetDiagnostics(diagnostics);
appState.SetFinishTime(finishTime);
return(appState);
}
protected internal virtual void CreateApplicationWithAMResource(ApplicationAttemptId
attId, string queue, string user, Resource amResource)
{
RMContext rmContext = resourceManager.GetRMContext();
RMApp rmApp = new RMAppImpl(attId.GetApplicationId(), rmContext, conf, null, null
, null, ApplicationSubmissionContext.NewInstance(null, null, null, null, null, false
, false, 0, amResource, null), null, null, 0, null, null, null);
rmContext.GetRMApps()[attId.GetApplicationId()] = rmApp;
AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(attId.GetApplicationId
(), queue, user);
scheduler.Handle(appAddedEvent);
AppAttemptAddedSchedulerEvent attempAddedEvent = new AppAttemptAddedSchedulerEvent
(attId, false);
scheduler.Handle(attempAddedEvent);
}
/// <exception cref="System.Exception"/>
public virtual void TestSubmitApplicationOnHA()
{
ApplicationSubmissionContext appContext = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ApplicationSubmissionContext>();
appContext.SetApplicationId(cluster.CreateFakeAppId());
ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ContainerLaunchContext>();
appContext.SetAMContainerSpec(amContainer);
Resource capability = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Resource>();
capability.SetMemory(10);
capability.SetVirtualCores(1);
appContext.SetResource(capability);
ApplicationId appId = client.SubmitApplication(appContext);
NUnit.Framework.Assert.IsTrue(GetActiveRM().GetRMContext().GetRMApps().Contains(appId
));
}
public static ApplicationSubmissionContext NewApplicationSubmissionContext(ApplicationId
applicationId, string applicationName, string queue, Priority priority, ContainerLaunchContext
amContainer, bool isUnmanagedAM, bool cancelTokensWhenComplete, int maxAppAttempts
, Resource resource, string applicationType)
{
ApplicationSubmissionContext context = recordFactory.NewRecordInstance <ApplicationSubmissionContext
>();
context.SetApplicationId(applicationId);
context.SetApplicationName(applicationName);
context.SetQueue(queue);
context.SetPriority(priority);
context.SetAMContainerSpec(amContainer);
context.SetUnmanagedAM(isUnmanagedAM);
context.SetCancelTokensWhenComplete(cancelTokensWhenComplete);
context.SetMaxAppAttempts(maxAppAttempts);
context.SetResource(resource);
context.SetApplicationType(applicationType);
return(context);
}
public virtual void SetUp()
{
long now = Runtime.CurrentTimeMillis();
rmContext = MockRMContext(1, now - 10);
ResourceScheduler scheduler = MockResourceScheduler();
Configuration conf = new Configuration();
ApplicationMasterService masterService = new ApplicationMasterService(rmContext,
scheduler);
appMonitor = new TestAppManager.TestRMAppManager(this, rmContext, new ClientToAMTokenSecretManagerInRM
(), scheduler, masterService, new ApplicationACLsManager(conf), conf);
appId = MockApps.NewAppID(1);
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(null);
asContext = recordFactory.NewRecordInstance <ApplicationSubmissionContext>();
asContext.SetApplicationId(appId);
asContext.SetAMContainerSpec(MockContainerLaunchContext(recordFactory));
asContext.SetResource(MockResource());
SetupDispatcher(rmContext, conf);
}
protected override void SetUp()
{
resourceMgrDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
conf = new YarnConfiguration();
conf.Set(YarnConfiguration.RmPrincipal, "mapred/host@REALM");
clientCache = new ClientCache(conf, resourceMgrDelegate);
clientCache = Org.Mockito.Mockito.Spy(clientCache);
yarnRunner = new YARNRunner(conf, resourceMgrDelegate, clientCache);
yarnRunner = Org.Mockito.Mockito.Spy(yarnRunner);
submissionContext = Org.Mockito.Mockito.Mock <ApplicationSubmissionContext>();
Org.Mockito.Mockito.DoAnswer(new _Answer_146(this)).When(yarnRunner).CreateApplicationSubmissionContext
(Matchers.Any <Configuration>(), Matchers.Any <string>(), Matchers.Any <Credentials
>());
appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 1);
jobId = TypeConverter.FromYarn(appId);
if (testWorkDir.Exists())
{
FileContext.GetLocalFSFileContext().Delete(new Path(testWorkDir.ToString()), true
);
}
testWorkDir.Mkdirs();
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.InvalidResourceRequestException
/// "/>
private ResourceRequest ValidateAndCreateResourceRequest(ApplicationSubmissionContext
submissionContext, bool isRecovery)
{
// Validation of the ApplicationSubmissionContext needs to be completed
// here. Only those fields that are dependent on RM's configuration are
// checked here as they have to be validated whether they are part of new
// submission or just being recovered.
// Check whether AM resource requirements are within required limits
if (!submissionContext.GetUnmanagedAM())
{
ResourceRequest amReq = submissionContext.GetAMContainerResourceRequest();
if (amReq == null)
{
amReq = BuilderUtils.NewResourceRequest(RMAppAttemptImpl.AmContainerPriority, ResourceRequest
.Any, submissionContext.GetResource(), 1);
}
// set label expression for AM container
if (null == amReq.GetNodeLabelExpression())
{
amReq.SetNodeLabelExpression(submissionContext.GetNodeLabelExpression());
}
try
{
SchedulerUtils.NormalizeAndValidateRequest(amReq, scheduler.GetMaximumResourceCapability
(), submissionContext.GetQueue(), scheduler, isRecovery, rmContext);
}
catch (InvalidResourceRequestException e)
{
Log.Warn("RM app submission failed in validating AM resource request" + " for application "
+ submissionContext.GetApplicationId(), e);
throw;
}
SchedulerUtils.NormalizeRequest(amReq, scheduler.GetResourceCalculator(), scheduler
.GetClusterResource(), scheduler.GetMinimumResourceCapability(), scheduler.GetMaximumResourceCapability
(), scheduler.GetMinimumResourceCapability());
return(amReq);
}
return(null);
}
/// <exception cref="System.Exception"/>
public virtual void TestAMProfiler()
{
JobConf jobConf = new JobConf();
jobConf.SetBoolean(MRJobConfig.MrAmProfile, true);
YARNRunner yarnRunner = new YARNRunner(jobConf);
ApplicationSubmissionContext submissionContext = BuildSubmitContext(yarnRunner, jobConf
);
ContainerLaunchContext containerSpec = submissionContext.GetAMContainerSpec();
IList <string> commands = containerSpec.GetCommands();
foreach (string command in commands)
{
if (command != null)
{
if (command.Contains(ProfileParams))
{
return;
}
}
}
throw new InvalidOperationException("Profiler opts not found!");
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
protected internal virtual void SubmitApplication(ApplicationSubmissionContext submissionContext
, long submitTime, string user)
{
ApplicationId applicationId = submissionContext.GetApplicationId();
RMAppImpl application = CreateAndPopulateNewRMApp(submissionContext, submitTime,
user, false);
ApplicationId appId = submissionContext.GetApplicationId();
if (UserGroupInformation.IsSecurityEnabled())
{
try
{
this.rmContext.GetDelegationTokenRenewer().AddApplicationAsync(appId, ParseCredentials
(submissionContext), submissionContext.GetCancelTokensWhenComplete(), application
.GetUser());
}
catch (Exception e)
{
Log.Warn("Unable to parse credentials.", e);
// Sending APP_REJECTED is fine, since we assume that the
// RMApp is in NEW state and thus we haven't yet informed the
// scheduler about the existence of the application
System.Diagnostics.Debug.Assert(application.GetState() == RMAppState.New);
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppEvent(applicationId
, RMAppEventType.AppRejected, e.Message));
throw RPCUtil.GetRemoteException(e);
}
}
else
{
// Dispatcher is not yet started at this time, so these START events
// enqueued should be guaranteed to be first processed when dispatcher
// gets started.
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppEvent(applicationId
, RMAppEventType.Start));
}
}
public virtual void Setup()
{
// start minicluster
conf = new YarnConfiguration();
yarnCluster = new MiniYARNCluster(typeof(TestAMRMClient).FullName, nodeCount, 1,
1);
yarnCluster.Init(conf);
yarnCluster.Start();
NUnit.Framework.Assert.IsNotNull(yarnCluster);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnCluster.GetServiceState
());
// start rm client
yarnClient = (YarnClientImpl)YarnClient.CreateYarnClient();
yarnClient.Init(conf);
yarnClient.Start();
NUnit.Framework.Assert.IsNotNull(yarnClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnClient.GetServiceState
());
// get node info
nodeReports = yarnClient.GetNodeReports(NodeState.Running);
// submit new app
ApplicationSubmissionContext appContext = yarnClient.CreateApplication().GetApplicationSubmissionContext
();
ApplicationId appId = appContext.GetApplicationId();
// set the application name
appContext.SetApplicationName("Test");
// Set the priority for the application master
Priority pri = Priority.NewInstance(0);
appContext.SetPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.SetQueue("default");
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ContainerLaunchContext>();
appContext.SetAMContainerSpec(amContainer);
// unmanaged AM
appContext.SetUnmanagedAM(true);
// Create the request to send to the applications manager
SubmitApplicationRequest appRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<SubmitApplicationRequest>();
appRequest.SetApplicationSubmissionContext(appContext);
// Submit the application to the applications manager
yarnClient.SubmitApplication(appContext);
// wait for app to start
int iterationsLeft = 30;
RMAppAttempt appAttempt = null;
while (iterationsLeft > 0)
{
ApplicationReport appReport = yarnClient.GetApplicationReport(appId);
if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted)
{
attemptId = appReport.GetCurrentApplicationAttemptId();
appAttempt = yarnCluster.GetResourceManager().GetRMContext().GetRMApps()[attemptId
.GetApplicationId()].GetCurrentAppAttempt();
while (true)
{
if (appAttempt.GetAppAttemptState() == RMAppAttemptState.Launched)
{
break;
}
}
break;
}
Sleep(1000);
--iterationsLeft;
}
if (iterationsLeft == 0)
{
NUnit.Framework.Assert.Fail("Application hasn't bee started");
}
// Just dig into the ResourceManager and get the AMRMToken just for the sake
// of testing.
UserGroupInformation.SetLoginUser(UserGroupInformation.CreateRemoteUser(UserGroupInformation
.GetCurrentUser().GetUserName()));
UserGroupInformation.GetCurrentUser().AddToken(appAttempt.GetAMRMToken());
//creating an instance NMTokenCase
nmTokenCache = new NMTokenCache();
// start am rm client
rmClient = (AMRMClientImpl <AMRMClient.ContainerRequest>)AMRMClient.CreateAMRMClient
<AMRMClient.ContainerRequest>();
//setting an instance NMTokenCase
rmClient.SetNMTokenCache(nmTokenCache);
rmClient.Init(conf);
rmClient.Start();
NUnit.Framework.Assert.IsNotNull(rmClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, rmClient.GetServiceState()
);
// start am nm client
nmClient = (NMClientImpl)NMClient.CreateNMClient();
//propagating the AMRMClient NMTokenCache instance
nmClient.SetNMTokenCache(rmClient.GetNMTokenCache());
nmClient.Init(conf);
nmClient.Start();
NUnit.Framework.Assert.IsNotNull(nmClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, nmClient.GetServiceState()
);
}
