public virtual void TestDeletionofStaging()
{
conf.Set(MRJobConfig.MapreduceJobDir, stagingJobDir);
fs = Org.Mockito.Mockito.Mock <FileSystem>();
Org.Mockito.Mockito.When(fs.Delete(Matchers.Any <Path>(), Matchers.AnyBoolean())).
ThenReturn(true);
//Staging Dir exists
string user = UserGroupInformation.GetCurrentUser().GetShortUserName();
Path stagingDir = MRApps.GetStagingAreaDir(conf, user);
Org.Mockito.Mockito.When(fs.Exists(stagingDir)).ThenReturn(true);
ApplicationId appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 0);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
JobId jobid = recordFactory.NewRecordInstance <JobId>();
jobid.SetAppId(appId);
ContainerAllocator mockAlloc = Org.Mockito.Mockito.Mock <ContainerAllocator>();
NUnit.Framework.Assert.IsTrue(MRJobConfig.DefaultMrAmMaxAttempts > 1);
MRAppMaster appMaster = new TestStagingCleanup.TestMRApp(this, attemptId, mockAlloc
, JobStateInternal.Running, MRJobConfig.DefaultMrAmMaxAttempts);
appMaster.Init(conf);
appMaster.Start();
appMaster.ShutDownJob();
//test whether notifyIsLastAMRetry called
NUnit.Framework.Assert.AreEqual(true, ((TestStagingCleanup.TestMRApp)appMaster).GetTestIsLastAMRetry
());
Org.Mockito.Mockito.Verify(fs).Delete(stagingJobPath, true);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestMRAppMasterMaxAppAttempts()
{
// No matter what's the maxAppAttempt or attempt id, the isLastRetry always
// equals to false
bool[] expectedBools = new bool[] { false, false, false };
string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
string containerIdStr = "container_1317529182569_0004_000002_1";
string userName = "******";
ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId
(applicationAttemptIdStr);
ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr);
JobConf conf = new JobConf();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
FilePath stagingDir = new FilePath(MRApps.GetStagingAreaDir(conf, userName).ToString
());
stagingDir.Mkdirs();
for (int i = 0; i < expectedBools.Length; ++i)
{
MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId
, "host", -1, -1, Runtime.CurrentTimeMillis(), false, true);
MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName);
NUnit.Framework.Assert.AreEqual("isLastAMRetry is correctly computed.", expectedBools
[i], appMaster.IsLastAMRetry());
}
}
// FIXME:
// Disabled this test because currently, when shutdown hook triggered at
// lastRetry in RM view, cleanup will not do. This should be supported after
// YARN-2261 completed
// @Test (timeout = 30000)
/// <exception cref="System.IO.IOException"/>
public virtual void TestDeletionofStagingOnKillLastTry()
{
conf.Set(MRJobConfig.MapreduceJobDir, stagingJobDir);
fs = Org.Mockito.Mockito.Mock <FileSystem>();
Org.Mockito.Mockito.When(fs.Delete(Matchers.Any <Path>(), Matchers.AnyBoolean())).
ThenReturn(true);
//Staging Dir exists
string user = UserGroupInformation.GetCurrentUser().GetShortUserName();
Path stagingDir = MRApps.GetStagingAreaDir(conf, user);
Org.Mockito.Mockito.When(fs.Exists(stagingDir)).ThenReturn(true);
ApplicationId appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 0);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
JobId jobid = recordFactory.NewRecordInstance <JobId>();
jobid.SetAppId(appId);
ContainerAllocator mockAlloc = Org.Mockito.Mockito.Mock <ContainerAllocator>();
MRAppMaster appMaster = new TestStagingCleanup.TestMRApp(this, attemptId, mockAlloc
);
//no retry
appMaster.Init(conf);
NUnit.Framework.Assert.IsTrue("appMaster.isLastAMRetry() is false", appMaster.IsLastAMRetry
());
//simulate the process being killed
MRAppMaster.MRAppMasterShutdownHook hook = new MRAppMaster.MRAppMasterShutdownHook
(appMaster);
hook.Run();
NUnit.Framework.Assert.IsTrue("MRAppMaster isn't stopped", appMaster.IsInState(Service.STATE
.Stopped));
Org.Mockito.Mockito.Verify(fs).Delete(stagingJobPath, true);
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
try
{
//Create the staging directory if it does not exist
string user = UserGroupInformation.GetCurrentUser().GetShortUserName();
Path stagingDir = MRApps.GetStagingAreaDir(conf, user);
FileSystem fs = GetFileSystem(conf);
fs.Mkdirs(stagingDir);
}
catch (Exception e)
{
throw new YarnRuntimeException("Error creating staging dir", e);
}
base.ServiceInit(conf);
if (this.clusterInfo != null)
{
GetContext().GetClusterInfo().SetMaxContainerCapability(this.clusterInfo.GetMaxContainerCapability
());
}
else
{
GetContext().GetClusterInfo().SetMaxContainerCapability(Resource.NewInstance(10240
, 1));
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual string GetStagingAreaDir()
{
// Path path = new Path(MRJobConstants.JOB_SUBMIT_DIR);
string user = UserGroupInformation.GetCurrentUser().GetShortUserName();
Path path = MRApps.GetStagingAreaDir(conf, user);
Log.Debug("getStagingAreaDir: dir=" + path);
return(path.ToString());
}
/// <summary>Gets the configured directory prefix for In Progress history files.</summary>
/// <param name="conf">the configuration for hte job</param>
/// <param name="jobId">the id of the job the history file is for.</param>
/// <returns>A string representation of the prefix.</returns>
/// <exception cref="System.IO.IOException"/>
public static string GetConfiguredHistoryStagingDirPrefix(Configuration conf, string
jobId)
{
string user = UserGroupInformation.GetCurrentUser().GetShortUserName();
Path stagingPath = MRApps.GetStagingAreaDir(conf, user);
Path path = new Path(stagingPath, jobId);
string logDir = path.ToString();
return(EnsurePathInDefaultFileSystem(logDir, conf));
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
if (overrideStart)
{
try
{
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
string user = ugi.GetShortUserName();
stagingDirPath = MRApps.GetStagingAreaDir(conf, user);
}
catch (Exception e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
}
else
{
base.ServiceStart();
}
}
/// <exception cref="System.IO.IOException"/>
private void TestDeletionofStagingOnUnregistrationFailure(int maxAttempts, bool shouldHaveDeleted
)
{
conf.Set(MRJobConfig.MapreduceJobDir, stagingJobDir);
fs = Org.Mockito.Mockito.Mock <FileSystem>();
Org.Mockito.Mockito.When(fs.Delete(Matchers.Any <Path>(), Matchers.AnyBoolean())).
ThenReturn(true);
//Staging Dir exists
string user = UserGroupInformation.GetCurrentUser().GetShortUserName();
Path stagingDir = MRApps.GetStagingAreaDir(conf, user);
Org.Mockito.Mockito.When(fs.Exists(stagingDir)).ThenReturn(true);
ApplicationId appId = ApplicationId.NewInstance(0, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
JobId jobid = recordFactory.NewRecordInstance <JobId>();
jobid.SetAppId(appId);
TestStagingCleanup.TestMRApp appMaster = new TestStagingCleanup.TestMRApp(this, attemptId
, null, JobStateInternal.Running, maxAttempts);
appMaster.crushUnregistration = true;
appMaster.Init(conf);
appMaster.Start();
appMaster.ShutDownJob();
((MRAppMaster.RunningAppContext)appMaster.GetContext()).ResetIsLastAMRetry();
if (shouldHaveDeleted)
{
NUnit.Framework.Assert.AreEqual(true, appMaster.IsLastAMRetry());
Org.Mockito.Mockito.Verify(fs).Delete(stagingJobPath, true);
}
else
{
NUnit.Framework.Assert.AreEqual(false, appMaster.IsLastAMRetry());
Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).Delete(stagingJobPath
, true);
}
}
public virtual void TestMRAppMasterCredentials()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
// Simulate credentials passed to AM via client->RM->NM
Credentials credentials = new Credentials();
byte[] identifier = Sharpen.Runtime.GetBytesForString("MyIdentifier");
byte[] password = Sharpen.Runtime.GetBytesForString("MyPassword");
Text kind = new Text("MyTokenKind");
Text service = new Text("host:port");
Org.Apache.Hadoop.Security.Token.Token<TokenIdentifier> myToken = new Org.Apache.Hadoop.Security.Token.Token
<TokenIdentifier>(identifier, password, kind, service);
Text tokenAlias = new Text("myToken");
credentials.AddToken(tokenAlias, myToken);
Text appTokenService = new Text("localhost:0");
Org.Apache.Hadoop.Security.Token.Token<AMRMTokenIdentifier> appToken = new Org.Apache.Hadoop.Security.Token.Token
<AMRMTokenIdentifier>(identifier, password, AMRMTokenIdentifier.KindName, appTokenService
);
credentials.AddToken(appTokenService, appToken);
Text keyAlias = new Text("mySecretKeyAlias");
credentials.AddSecretKey(keyAlias, Sharpen.Runtime.GetBytesForString("mySecretKey"
));
Org.Apache.Hadoop.Security.Token.Token<TokenIdentifier> storedToken = credentials
.GetToken(tokenAlias);
JobConf conf = new JobConf();
Path tokenFilePath = new Path(testDir.GetAbsolutePath(), "tokens-file");
IDictionary<string, string> newEnv = new Dictionary<string, string>();
newEnv[UserGroupInformation.HadoopTokenFileLocation] = tokenFilePath.ToUri().GetPath
();
SetNewEnvironmentHack(newEnv);
credentials.WriteTokenStorageFile(tokenFilePath, conf);
ApplicationId appId = ApplicationId.NewInstance(12345, 56);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(appId
, 1);
ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 546);
string userName = UserGroupInformation.GetCurrentUser().GetShortUserName();
// Create staging dir, so MRAppMaster doesn't barf.
FilePath stagingDir = new FilePath(MRApps.GetStagingAreaDir(conf, userName).ToString
());
stagingDir.Mkdirs();
// Set login-user to null as that is how real world MRApp starts with.
// This is null is the reason why token-file is read by UGI.
UserGroupInformation.SetLoginUser(null);
MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId
, "host", -1, -1, Runtime.CurrentTimeMillis(), false, true);
MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName);
// Now validate the task credentials
Credentials appMasterCreds = appMaster.GetCredentials();
NUnit.Framework.Assert.IsNotNull(appMasterCreds);
NUnit.Framework.Assert.AreEqual(1, appMasterCreds.NumberOfSecretKeys());
NUnit.Framework.Assert.AreEqual(1, appMasterCreds.NumberOfTokens());
// Validate the tokens - app token should not be present
Org.Apache.Hadoop.Security.Token.Token<TokenIdentifier> usedToken = appMasterCreds
.GetToken(tokenAlias);
NUnit.Framework.Assert.IsNotNull(usedToken);
NUnit.Framework.Assert.AreEqual(storedToken, usedToken);
// Validate the keys
byte[] usedKey = appMasterCreds.GetSecretKey(keyAlias);
NUnit.Framework.Assert.IsNotNull(usedKey);
NUnit.Framework.Assert.AreEqual("mySecretKey", Sharpen.Runtime.GetStringForBytes(
usedKey));
// The credentials should also be added to conf so that OuputCommitter can
// access it - app token should not be present
Credentials confCredentials = conf.GetCredentials();
NUnit.Framework.Assert.AreEqual(1, confCredentials.NumberOfSecretKeys());
NUnit.Framework.Assert.AreEqual(1, confCredentials.NumberOfTokens());
NUnit.Framework.Assert.AreEqual(storedToken, confCredentials.GetToken(tokenAlias)
);
NUnit.Framework.Assert.AreEqual("mySecretKey", Sharpen.Runtime.GetStringForBytes(
confCredentials.GetSecretKey(keyAlias)));
// Verify the AM's ugi - app token should be present
Credentials ugiCredentials = appMaster.GetUgi().GetCredentials();
NUnit.Framework.Assert.AreEqual(1, ugiCredentials.NumberOfSecretKeys());
NUnit.Framework.Assert.AreEqual(2, ugiCredentials.NumberOfTokens());
NUnit.Framework.Assert.AreEqual(storedToken, ugiCredentials.GetToken(tokenAlias));
NUnit.Framework.Assert.AreEqual(appToken, ugiCredentials.GetToken(appTokenService
));
NUnit.Framework.Assert.AreEqual("mySecretKey", Sharpen.Runtime.GetStringForBytes(
ugiCredentials.GetSecretKey(keyAlias)));
}
