/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestMRAppMasterMaxAppAttempts()
{
// No matter what's the maxAppAttempt or attempt id, the isLastRetry always
// equals to false
bool[] expectedBools = new bool[] { false, false, false };
string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
string containerIdStr = "container_1317529182569_0004_000002_1";
string userName = "******";
ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId
(applicationAttemptIdStr);
ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr);
JobConf conf = new JobConf();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
FilePath stagingDir = new FilePath(MRApps.GetStagingAreaDir(conf, userName).ToString
());
stagingDir.Mkdirs();
for (int i = 0; i < expectedBools.Length; ++i)
{
MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId
, "host", -1, -1, Runtime.CurrentTimeMillis(), false, true);
MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName);
NUnit.Framework.Assert.AreEqual("isLastAMRetry is correctly computed.", expectedBools
[i], appMaster.IsLastAMRetry());
}
}
public virtual void TestMRAppMasterForDifferentUser()
{
string applicationAttemptIdStr = "appattempt_1317529182569_0004_000001";
string containerIdStr = "container_1317529182569_0004_000001_1";
string userName = "******";
ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId
(applicationAttemptIdStr);
ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr);
MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId
, "host", -1, -1, Runtime.CurrentTimeMillis());
JobConf conf = new JobConf();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName);
Path userPath = new Path(stagingDir, userName);
Path userStagingPath = new Path(userPath, ".staging");
NUnit.Framework.Assert.AreEqual(userStagingPath.ToString(), appMaster.stagingDirPath
.ToString());
}
public virtual void TestMRAppMasterSuccessLock()
{
string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
string containerIdStr = "container_1317529182569_0004_000002_1";
string userName = "******";
JobConf conf = new JobConf();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId
(applicationAttemptIdStr);
JobId jobId = TypeConverter.ToYarn(TypeConverter.FromYarn(applicationAttemptId.GetApplicationId
()));
Path start = MRApps.GetStartJobCommitFile(conf, userName, jobId);
Path end = MRApps.GetEndJobCommitSuccessFile(conf, userName, jobId);
FileSystem fs = FileSystem.Get(conf);
fs.Create(start).Close();
fs.Create(end).Close();
ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr);
MRAppMaster appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host"
, -1, -1, Runtime.CurrentTimeMillis(), false, false);
bool caught = false;
try
{
MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName);
}
catch (IOException e)
{
//The IO Exception is expected
Log.Info("Caught expected Exception", e);
caught = true;
}
NUnit.Framework.Assert.IsTrue(caught);
NUnit.Framework.Assert.IsTrue(appMaster.errorHappenedShutDown);
NUnit.Framework.Assert.AreEqual(JobStateInternal.Succeeded, appMaster.forcedState
);
appMaster.Stop();
// verify the final status is SUCCEEDED
VerifyFailedStatus((MRAppMasterTest)appMaster, "SUCCEEDED");
}
public virtual void TestMRAppMasterMissingStaging()
{
string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
string containerIdStr = "container_1317529182569_0004_000002_1";
string userName = "******";
JobConf conf = new JobConf();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId
(applicationAttemptIdStr);
//Delete the staging directory
FilePath dir = new FilePath(stagingDir);
if (dir.Exists())
{
FileUtils.DeleteDirectory(dir);
}
ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr);
MRAppMaster appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host"
, -1, -1, Runtime.CurrentTimeMillis(), false, false);
bool caught = false;
try
{
MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName);
}
catch (IOException e)
{
//The IO Exception is expected
Log.Info("Caught expected Exception", e);
caught = true;
}
NUnit.Framework.Assert.IsTrue(caught);
NUnit.Framework.Assert.IsTrue(appMaster.errorHappenedShutDown);
//Copying the history file is disabled, but it is not really visible from
//here
NUnit.Framework.Assert.AreEqual(JobStateInternal.Error, appMaster.forcedState);
appMaster.Stop();
}
public virtual void TestMRAppMasterCredentials()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
// Simulate credentials passed to AM via client->RM->NM
Credentials credentials = new Credentials();
byte[] identifier = Sharpen.Runtime.GetBytesForString("MyIdentifier");
byte[] password = Sharpen.Runtime.GetBytesForString("MyPassword");
Text kind = new Text("MyTokenKind");
Text service = new Text("host:port");
Org.Apache.Hadoop.Security.Token.Token<TokenIdentifier> myToken = new Org.Apache.Hadoop.Security.Token.Token
<TokenIdentifier>(identifier, password, kind, service);
Text tokenAlias = new Text("myToken");
credentials.AddToken(tokenAlias, myToken);
Text appTokenService = new Text("localhost:0");
Org.Apache.Hadoop.Security.Token.Token<AMRMTokenIdentifier> appToken = new Org.Apache.Hadoop.Security.Token.Token
<AMRMTokenIdentifier>(identifier, password, AMRMTokenIdentifier.KindName, appTokenService
);
credentials.AddToken(appTokenService, appToken);
Text keyAlias = new Text("mySecretKeyAlias");
credentials.AddSecretKey(keyAlias, Sharpen.Runtime.GetBytesForString("mySecretKey"
));
Org.Apache.Hadoop.Security.Token.Token<TokenIdentifier> storedToken = credentials
.GetToken(tokenAlias);
JobConf conf = new JobConf();
Path tokenFilePath = new Path(testDir.GetAbsolutePath(), "tokens-file");
IDictionary<string, string> newEnv = new Dictionary<string, string>();
newEnv[UserGroupInformation.HadoopTokenFileLocation] = tokenFilePath.ToUri().GetPath
();
SetNewEnvironmentHack(newEnv);
credentials.WriteTokenStorageFile(tokenFilePath, conf);
ApplicationId appId = ApplicationId.NewInstance(12345, 56);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(appId
, 1);
ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 546);
string userName = UserGroupInformation.GetCurrentUser().GetShortUserName();
// Create staging dir, so MRAppMaster doesn't barf.
FilePath stagingDir = new FilePath(MRApps.GetStagingAreaDir(conf, userName).ToString
());
stagingDir.Mkdirs();
// Set login-user to null as that is how real world MRApp starts with.
// This is null is the reason why token-file is read by UGI.
UserGroupInformation.SetLoginUser(null);
MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId
, "host", -1, -1, Runtime.CurrentTimeMillis(), false, true);
MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName);
// Now validate the task credentials
Credentials appMasterCreds = appMaster.GetCredentials();
NUnit.Framework.Assert.IsNotNull(appMasterCreds);
NUnit.Framework.Assert.AreEqual(1, appMasterCreds.NumberOfSecretKeys());
NUnit.Framework.Assert.AreEqual(1, appMasterCreds.NumberOfTokens());
// Validate the tokens - app token should not be present
Org.Apache.Hadoop.Security.Token.Token<TokenIdentifier> usedToken = appMasterCreds
.GetToken(tokenAlias);
NUnit.Framework.Assert.IsNotNull(usedToken);
NUnit.Framework.Assert.AreEqual(storedToken, usedToken);
// Validate the keys
byte[] usedKey = appMasterCreds.GetSecretKey(keyAlias);
NUnit.Framework.Assert.IsNotNull(usedKey);
NUnit.Framework.Assert.AreEqual("mySecretKey", Sharpen.Runtime.GetStringForBytes(
usedKey));
// The credentials should also be added to conf so that OuputCommitter can
// access it - app token should not be present
Credentials confCredentials = conf.GetCredentials();
NUnit.Framework.Assert.AreEqual(1, confCredentials.NumberOfSecretKeys());
NUnit.Framework.Assert.AreEqual(1, confCredentials.NumberOfTokens());
NUnit.Framework.Assert.AreEqual(storedToken, confCredentials.GetToken(tokenAlias)
);
NUnit.Framework.Assert.AreEqual("mySecretKey", Sharpen.Runtime.GetStringForBytes(
confCredentials.GetSecretKey(keyAlias)));
// Verify the AM's ugi - app token should be present
Credentials ugiCredentials = appMaster.GetUgi().GetCredentials();
NUnit.Framework.Assert.AreEqual(1, ugiCredentials.NumberOfSecretKeys());
NUnit.Framework.Assert.AreEqual(2, ugiCredentials.NumberOfTokens());
NUnit.Framework.Assert.AreEqual(storedToken, ugiCredentials.GetToken(tokenAlias));
NUnit.Framework.Assert.AreEqual(appToken, ugiCredentials.GetToken(appTokenService
));
NUnit.Framework.Assert.AreEqual("mySecretKey", Sharpen.Runtime.GetStringForBytes(
ugiCredentials.GetSecretKey(keyAlias)));
}
