/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private RMAppImpl CreateAndPopulateNewRMApp(ApplicationSubmissionContext submissionContext
, long submitTime, string user, bool isRecovery)
{
ApplicationId applicationId = submissionContext.GetApplicationId();
ResourceRequest amReq = ValidateAndCreateResourceRequest(submissionContext, isRecovery
);
// Create RMApp
RMAppImpl application = new RMAppImpl(applicationId, rmContext, this.conf, submissionContext
.GetApplicationName(), user, submissionContext.GetQueue(), submissionContext, this
.scheduler, this.masterService, submitTime, submissionContext.GetApplicationType
(), submissionContext.GetApplicationTags(), amReq);
// Concurrent app submissions with same applicationId will fail here
// Concurrent app submissions with different applicationIds will not
// influence each other
if (rmContext.GetRMApps().PutIfAbsent(applicationId, application) != null)
{
string message = "Application with id " + applicationId + " is already present! Cannot add a duplicate!";
Log.Warn(message);
throw new YarnException(message);
}
// Inform the ACLs Manager
this.applicationACLsManager.AddApplication(applicationId, submissionContext.GetAMContainerSpec
().GetApplicationACLs());
string appViewACLs = submissionContext.GetAMContainerSpec().GetApplicationACLs()[
ApplicationAccessType.ViewApp];
rmContext.GetSystemMetricsPublisher().AppACLsUpdated(application, appViewACLs, Runtime
.CurrentTimeMillis());
return(application);
}
public virtual void TestAMStandardEnv()
{
string AdminLibPath = "foo";
string UserLibPath = "bar";
string UserShell = "shell";
JobConf jobConf = new JobConf();
jobConf.Set(MRJobConfig.MrAmAdminUserEnv, "LD_LIBRARY_PATH=" + AdminLibPath);
jobConf.Set(MRJobConfig.MrAmEnv, "LD_LIBRARY_PATH=" + UserLibPath);
jobConf.Set(MRJobConfig.MapredAdminUserShell, UserShell);
YARNRunner yarnRunner = new YARNRunner(jobConf);
ApplicationSubmissionContext appSubCtx = BuildSubmitContext(yarnRunner, jobConf);
// make sure PWD is first in the lib path
ContainerLaunchContext clc = appSubCtx.GetAMContainerSpec();
IDictionary <string, string> env = clc.GetEnvironment();
string libPath = env[ApplicationConstants.Environment.LdLibraryPath.ToString()];
NUnit.Framework.Assert.IsNotNull("LD_LIBRARY_PATH not set", libPath);
string cps = jobConf.GetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, MRConfig
.DefaultMapreduceAppSubmissionCrossPlatform) ? ApplicationConstants.ClassPathSeparator
: FilePath.pathSeparator;
NUnit.Framework.Assert.AreEqual("Bad AM LD_LIBRARY_PATH setting", MRApps.CrossPlatformifyMREnv
(conf, ApplicationConstants.Environment.Pwd) + cps + AdminLibPath + cps + UserLibPath
, libPath);
// make sure SHELL is set
string shell = env[ApplicationConstants.Environment.Shell.ToString()];
NUnit.Framework.Assert.IsNotNull("SHELL not set", shell);
NUnit.Framework.Assert.AreEqual("Bad SHELL setting", UserShell, shell);
}
/// <exception cref="System.Exception"/>
public virtual void TestAMAdminCommandOpts()
{
JobConf jobConf = new JobConf();
jobConf.Set(MRJobConfig.MrAmAdminCommandOpts, "-Djava.net.preferIPv4Stack=true");
jobConf.Set(MRJobConfig.MrAmCommandOpts, "-Xmx1024m");
YARNRunner yarnRunner = new YARNRunner(jobConf);
ApplicationSubmissionContext submissionContext = BuildSubmitContext(yarnRunner, jobConf
);
ContainerLaunchContext containerSpec = submissionContext.GetAMContainerSpec();
IList <string> commands = containerSpec.GetCommands();
int index = 0;
int adminIndex = 0;
int adminPos = -1;
int userIndex = 0;
int userPos = -1;
int tmpDirPos = -1;
foreach (string command in commands)
{
if (command != null)
{
NUnit.Framework.Assert.IsFalse("Profiler should be disabled by default", command.
Contains(ProfileParams));
adminPos = command.IndexOf("-Djava.net.preferIPv4Stack=true");
if (adminPos >= 0)
{
adminIndex = index;
}
userPos = command.IndexOf("-Xmx1024m");
if (userPos >= 0)
{
userIndex = index;
}
tmpDirPos = command.IndexOf("-Djava.io.tmpdir=");
}
index++;
}
// Check java.io.tmpdir opts are set in the commands
NUnit.Framework.Assert.IsTrue("java.io.tmpdir is not set for AM", tmpDirPos > 0);
// Check both admin java opts and user java opts are in the commands
NUnit.Framework.Assert.IsTrue("AM admin command opts not in the commands.", adminPos
> 0);
NUnit.Framework.Assert.IsTrue("AM user command opts not in the commands.", userPos
> 0);
// Check the admin java opts is before user java opts in the commands
if (adminIndex == userIndex)
{
NUnit.Framework.Assert.IsTrue("AM admin command opts is after user command opts."
, adminPos < userPos);
}
else
{
NUnit.Framework.Assert.IsTrue("AM admin command opts is after user command opts."
, adminIndex < userIndex);
}
}
// TODO: Don't create again and again.
/// <exception cref="System.IO.IOException"/>
private ContainerLaunchContext CreateAMContainerLaunchContext(ApplicationSubmissionContext
applicationMasterContext, ContainerId containerID)
{
// Construct the actual Container
ContainerLaunchContext container = applicationMasterContext.GetAMContainerSpec();
Log.Info("Command to launch container " + containerID + " : " + StringUtils.ArrayToString
(Sharpen.Collections.ToArray(container.GetCommands(), new string[0])));
// Finalize the container
SetupTokens(container, containerID);
return(container);
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual Credentials ParseCredentials(ApplicationSubmissionContext
application)
{
Credentials credentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
ByteBuffer tokens = application.GetAMContainerSpec().GetTokens();
if (tokens != null)
{
dibb.Reset(tokens);
credentials.ReadTokenStorageStream(dibb);
tokens.Rewind();
}
return(credentials);
}
/// <exception cref="System.Exception"/>
public virtual void TestAMProfiler()
{
JobConf jobConf = new JobConf();
jobConf.SetBoolean(MRJobConfig.MrAmProfile, true);
YARNRunner yarnRunner = new YARNRunner(jobConf);
ApplicationSubmissionContext submissionContext = BuildSubmitContext(yarnRunner, jobConf
);
ContainerLaunchContext containerSpec = submissionContext.GetAMContainerSpec();
IList <string> commands = containerSpec.GetCommands();
foreach (string command in commands)
{
if (command != null)
{
if (command.Contains(ProfileParams))
{
return;
}
}
}
throw new InvalidOperationException("Profiler opts not found!");
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public override ApplicationId SubmitApplication(ApplicationSubmissionContext appContext
)
{
ApplicationId applicationId = appContext.GetApplicationId();
if (applicationId == null)
{
throw new ApplicationIdNotProvidedException("ApplicationId is not provided in ApplicationSubmissionContext"
);
}
SubmitApplicationRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <
SubmitApplicationRequest>();
request.SetApplicationSubmissionContext(appContext);
// Automatically add the timeline DT into the CLC
// Only when the security and the timeline service are both enabled
if (IsSecurityEnabled() && timelineServiceEnabled)
{
AddTimelineDelegationToken(appContext.GetAMContainerSpec());
}
//TODO: YARN-1763:Handle RM failovers during the submitApplication call.
rmClient.SubmitApplication(request);
int pollCount = 0;
long startTime = Runtime.CurrentTimeMillis();
EnumSet <YarnApplicationState> waitingStates = EnumSet.Of(YarnApplicationState.New
, YarnApplicationState.NewSaving, YarnApplicationState.Submitted);
EnumSet <YarnApplicationState> failToSubmitStates = EnumSet.Of(YarnApplicationState
.Failed, YarnApplicationState.Killed);
while (true)
{
try
{
ApplicationReport appReport = GetApplicationReport(applicationId);
YarnApplicationState state = appReport.GetYarnApplicationState();
if (!waitingStates.Contains(state))
{
if (failToSubmitStates.Contains(state))
{
throw new YarnException("Failed to submit " + applicationId + " to YARN : " + appReport
.GetDiagnostics());
}
Log.Info("Submitted application " + applicationId);
break;
}
long elapsedMillis = Runtime.CurrentTimeMillis() - startTime;
if (EnforceAsyncAPITimeout() && elapsedMillis >= asyncApiPollTimeoutMillis)
{
throw new YarnException("Timed out while waiting for application " + applicationId
+ " to be submitted successfully");
}
// Notify the client through the log every 10 poll, in case the client
// is blocked here too long.
if (++pollCount % 10 == 0)
{
Log.Info("Application submission is not finished, " + "submitted application " +
applicationId + " is still in " + state);
}
try
{
Sharpen.Thread.Sleep(submitPollIntervalMillis);
}
catch (Exception)
{
Log.Error("Interrupted while waiting for application " + applicationId + " to be successfully submitted."
);
}
}
catch (ApplicationNotFoundException)
{
// FailOver or RM restart happens before RMStateStore saves
// ApplicationState
Log.Info("Re-submit application " + applicationId + "with the " + "same ApplicationSubmissionContext"
);
rmClient.SubmitApplication(request);
}
}
return(applicationId);
}
