public static ApplicationAttemptId GetMockApplicationAttemptId(int appId, int attemptId
)
{
ApplicationId applicationId = BuilderUtils.NewApplicationId(0l, appId);
ApplicationAttemptId applicationAttemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptId
>();
Org.Mockito.Mockito.When(applicationAttemptId.GetApplicationId()).ThenReturn(applicationId
);
Org.Mockito.Mockito.When(applicationAttemptId.GetAttemptId()).ThenReturn(attemptId
);
return(applicationAttemptId);
}
/// <exception cref="System.IO.IOException"/>
public static Path GetPreviousJobHistoryPath(Configuration conf, ApplicationAttemptId
applicationAttemptId)
{
string jobId = TypeConverter.FromYarn(applicationAttemptId.GetApplicationId()).ToString
();
string jobhistoryDir = JobHistoryUtils.GetConfiguredHistoryStagingDirPrefix(conf,
jobId);
Path histDirPath = FileContext.GetFileContext(conf).MakeQualified(new Path(jobhistoryDir
));
FileContext fc = FileContext.GetFileContext(histDirPath.ToUri(), conf);
return(fc.MakeQualified(JobHistoryUtils.GetStagingJobHistoryFile(histDirPath, jobId
, (applicationAttemptId.GetAttemptId() - 1))));
}
internal static ContainerId GetMockContainer(long id)
{
ApplicationId appId = Org.Mockito.Mockito.Mock <ApplicationId>();
Org.Mockito.Mockito.When(appId.GetClusterTimestamp()).ThenReturn(314159265L);
Org.Mockito.Mockito.When(appId.GetId()).ThenReturn(3);
ApplicationAttemptId appAttemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptId
>();
Org.Mockito.Mockito.When(appAttemptId.GetApplicationId()).ThenReturn(appId);
Org.Mockito.Mockito.When(appAttemptId.GetAttemptId()).ThenReturn(0);
ContainerId container = Org.Mockito.Mockito.Mock <ContainerId>();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(id);
Org.Mockito.Mockito.When(container.GetApplicationAttemptId()).ThenReturn(appAttemptId
);
return(container);
}
/// <exception cref="System.IO.IOException"/>
private static void ConfigureTask(JobConf job, Task task, Credentials credentials
, Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jt)
{
job.SetCredentials(credentials);
ApplicationAttemptId appAttemptId = ConverterUtils.ToContainerId(Runtime.Getenv(ApplicationConstants.Environment
.ContainerId.ToString())).GetApplicationAttemptId();
Log.Debug("APPLICATION_ATTEMPT_ID: " + appAttemptId);
// Set it in conf, so as to be able to be used the the OutputCommitter.
job.SetInt(MRJobConfig.ApplicationAttemptId, appAttemptId.GetAttemptId());
// set tcp nodelay
job.SetBoolean("ipc.client.tcpnodelay", true);
job.SetClass(MRConfig.TaskLocalOutputClass, typeof(YarnOutputFiles), typeof(MapOutputFile
));
// set the jobToken and shuffle secrets into task
task.SetJobTokenSecret(JobTokenSecretManager.CreateSecretKey(jt.GetPassword()));
byte[] shuffleSecret = TokenCache.GetShuffleSecretKey(credentials);
if (shuffleSecret == null)
{
Log.Warn("Shuffle secret missing from task credentials." + " Using job token secret as shuffle secret."
);
shuffleSecret = jt.GetPassword();
}
task.SetShuffleSecret(JobTokenSecretManager.CreateSecretKey(shuffleSecret));
// setup the child's MRConfig.LOCAL_DIR.
ConfigureLocalDirs(task, job);
// setup the child's attempt directories
// Do the task-type specific localization
task.LocalizeConfiguration(job);
// Set up the DistributedCache related configs
MRApps.SetupDistributedCacheLocal(job);
// Overwrite the localized task jobconf which is linked to in the current
// work-dir.
Path localTaskFile = new Path(MRJobConfig.JobConfFile);
WriteLocalJobFile(localTaskFile, job);
task.SetJobFile(localTaskFile.ToString());
task.SetConf(job);
}
/// <exception cref="System.IO.IOException"/>
private ApplicationAttemptHistoryData GetLastAttempt(ApplicationId appId)
{
IDictionary <ApplicationAttemptId, ApplicationAttemptHistoryData> attempts = historyStore
.GetApplicationAttempts(appId);
ApplicationAttemptId prevMaxAttemptId = null;
foreach (ApplicationAttemptId attemptId in attempts.Keys)
{
if (prevMaxAttemptId == null)
{
prevMaxAttemptId = attemptId;
}
else
{
if (prevMaxAttemptId.GetAttemptId() < attemptId.GetAttemptId())
{
prevMaxAttemptId = attemptId;
}
}
}
return(attempts[prevMaxAttemptId]);
}
/// <exception cref="System.Exception"/>
private void TestNMTokens(Configuration conf)
{
NMTokenSecretManagerInRM nmTokenSecretManagerRM = yarnCluster.GetResourceManager(
).GetRMContext().GetNMTokenSecretManager();
NMTokenSecretManagerInNM nmTokenSecretManagerNM = yarnCluster.GetNodeManager(0).GetNMContext
().GetNMTokenSecretManager();
RMContainerTokenSecretManager containerTokenSecretManager = yarnCluster.GetResourceManager
().GetRMContext().GetContainerTokenSecretManager();
NodeManager nm = yarnCluster.GetNodeManager(0);
WaitForNMToReceiveNMTokenKey(nmTokenSecretManagerNM, nm);
// Both id should be equal.
NUnit.Framework.Assert.AreEqual(nmTokenSecretManagerNM.GetCurrentKey().GetKeyId()
, nmTokenSecretManagerRM.GetCurrentKey().GetKeyId());
/*
* Below cases should be tested.
* 1) If Invalid NMToken is used then it should be rejected.
* 2) If valid NMToken but belonging to another Node is used then that
* too should be rejected.
* 3) NMToken for say appAttempt-1 is used for starting/stopping/retrieving
* status for container with containerId for say appAttempt-2 should
* be rejected.
* 4) After start container call is successful nmtoken should have been
* saved in NMTokenSecretManagerInNM.
* 5) If start container call was successful (no matter if container is
* still running or not), appAttempt->NMToken should be present in
* NMTokenSecretManagerInNM's cache. Any future getContainerStatus call
* for containerId belonging to that application attempt using
* applicationAttempt's older nmToken should not get any invalid
* nmToken error. (This can be best tested if we roll over NMToken
* master key twice).
*/
YarnRPC rpc = YarnRPC.Create(conf);
string user = "******";
Resource r = Resource.NewInstance(1024, 1);
ApplicationId appId = ApplicationId.NewInstance(1, 1);
ApplicationAttemptId validAppAttemptId = ApplicationAttemptId.NewInstance(appId,
1);
ContainerId validContainerId = ContainerId.NewContainerId(validAppAttemptId, 0);
NodeId validNode = yarnCluster.GetNodeManager(0).GetNMContext().GetNodeId();
NodeId invalidNode = NodeId.NewInstance("InvalidHost", 1234);
Token validNMToken = nmTokenSecretManagerRM.CreateNMToken(validAppAttemptId, validNode
, user);
Token validContainerToken = containerTokenSecretManager.CreateContainerToken(validContainerId
, validNode, user, r, Priority.NewInstance(10), 1234);
ContainerTokenIdentifier identifier = BuilderUtils.NewContainerTokenIdentifier(validContainerToken
);
NUnit.Framework.Assert.AreEqual(Priority.NewInstance(10), identifier.GetPriority(
));
NUnit.Framework.Assert.AreEqual(1234, identifier.GetCreationTime());
StringBuilder sb;
// testInvalidNMToken ... creating NMToken using different secret manager.
NMTokenSecretManagerInRM tempManager = new NMTokenSecretManagerInRM(conf);
tempManager.RollMasterKey();
do
{
tempManager.RollMasterKey();
tempManager.ActivateNextMasterKey();
}while (tempManager.GetCurrentKey().GetKeyId() == nmTokenSecretManagerRM.GetCurrentKey
().GetKeyId());
// Making sure key id is different.
// Testing that NM rejects the requests when we don't send any token.
if (UserGroupInformation.IsSecurityEnabled())
{
sb = new StringBuilder("Client cannot authenticate via:[TOKEN]");
}
else
{
sb = new StringBuilder("SIMPLE authentication is not enabled. Available:[TOKEN]"
);
}
string errorMsg = TestStartContainer(rpc, validAppAttemptId, validNode, validContainerToken
, null, true);
NUnit.Framework.Assert.IsTrue(errorMsg.Contains(sb.ToString()));
Token invalidNMToken = tempManager.CreateNMToken(validAppAttemptId, validNode, user
);
sb = new StringBuilder("Given NMToken for application : ");
sb.Append(validAppAttemptId.ToString()).Append(" seems to have been generated illegally."
);
NUnit.Framework.Assert.IsTrue(sb.ToString().Contains(TestStartContainer(rpc, validAppAttemptId
, validNode, validContainerToken, invalidNMToken, true)));
// valid NMToken but belonging to other node
invalidNMToken = nmTokenSecretManagerRM.CreateNMToken(validAppAttemptId, invalidNode
, user);
sb = new StringBuilder("Given NMToken for application : ");
sb.Append(validAppAttemptId).Append(" is not valid for current node manager.expected : "
).Append(validNode.ToString()).Append(" found : ").Append(invalidNode.ToString()
);
NUnit.Framework.Assert.IsTrue(sb.ToString().Contains(TestStartContainer(rpc, validAppAttemptId
, validNode, validContainerToken, invalidNMToken, true)));
// using correct tokens. nmtoken for app attempt should get saved.
conf.SetInt(YarnConfiguration.RmContainerAllocExpiryIntervalMs, 4 * 60 * 1000);
validContainerToken = containerTokenSecretManager.CreateContainerToken(validContainerId
, validNode, user, r, Priority.NewInstance(0), 0);
NUnit.Framework.Assert.IsTrue(TestStartContainer(rpc, validAppAttemptId, validNode
, validContainerToken, validNMToken, false).IsEmpty());
NUnit.Framework.Assert.IsTrue(nmTokenSecretManagerNM.IsAppAttemptNMTokenKeyPresent
(validAppAttemptId));
// using a new compatible version nmtoken, expect container can be started
// successfully.
ApplicationAttemptId validAppAttemptId2 = ApplicationAttemptId.NewInstance(appId,
2);
ContainerId validContainerId2 = ContainerId.NewContainerId(validAppAttemptId2, 0);
Token validContainerToken2 = containerTokenSecretManager.CreateContainerToken(validContainerId2
, validNode, user, r, Priority.NewInstance(0), 0);
Token validNMToken2 = nmTokenSecretManagerRM.CreateNMToken(validAppAttemptId2, validNode
, user);
// First, get a new NMTokenIdentifier.
NMTokenIdentifier newIdentifier = new NMTokenIdentifier();
byte[] tokenIdentifierContent = ((byte[])validNMToken2.GetIdentifier().Array());
DataInputBuffer dib = new DataInputBuffer();
dib.Reset(tokenIdentifierContent, tokenIdentifierContent.Length);
newIdentifier.ReadFields(dib);
// Then, generate a new version NMTokenIdentifier (NMTokenIdentifierNewForTest)
// with additional field of message.
NMTokenIdentifierNewForTest newVersionIdentifier = new NMTokenIdentifierNewForTest
(newIdentifier, "message");
// check new version NMTokenIdentifier has correct info.
NUnit.Framework.Assert.AreEqual("The ApplicationAttemptId is changed after set to "
+ "newVersionIdentifier", validAppAttemptId2.GetAttemptId(), newVersionIdentifier
.GetApplicationAttemptId().GetAttemptId());
NUnit.Framework.Assert.AreEqual("The message is changed after set to newVersionIdentifier"
, "message", newVersionIdentifier.GetMessage());
NUnit.Framework.Assert.AreEqual("The NodeId is changed after set to newVersionIdentifier"
, validNode, newVersionIdentifier.GetNodeId());
// create new Token based on new version NMTokenIdentifier.
Token newVersionedNMToken = BaseNMTokenSecretManager.NewInstance(nmTokenSecretManagerRM
.RetrievePassword(newVersionIdentifier), newVersionIdentifier);
// Verify startContainer is successful and no exception is thrown.
NUnit.Framework.Assert.IsTrue(TestStartContainer(rpc, validAppAttemptId2, validNode
, validContainerToken2, newVersionedNMToken, false).IsEmpty());
NUnit.Framework.Assert.IsTrue(nmTokenSecretManagerNM.IsAppAttemptNMTokenKeyPresent
(validAppAttemptId2));
//Now lets wait till container finishes and is removed from node manager.
WaitForContainerToFinishOnNM(validContainerId);
sb = new StringBuilder("Attempt to relaunch the same container with id ");
sb.Append(validContainerId);
NUnit.Framework.Assert.IsTrue(TestStartContainer(rpc, validAppAttemptId, validNode
, validContainerToken, validNMToken, true).Contains(sb.ToString()));
// Container is removed from node manager's memory by this time.
// trying to stop the container. It should not throw any exception.
TestStopContainer(rpc, validAppAttemptId, validNode, validContainerId, validNMToken
, false);
// Rolling over master key twice so that we can check whether older keys
// are used for authentication.
RollNMTokenMasterKey(nmTokenSecretManagerRM, nmTokenSecretManagerNM);
// Key rolled over once.. rolling over again
RollNMTokenMasterKey(nmTokenSecretManagerRM, nmTokenSecretManagerNM);
// trying get container status. Now saved nmToken should be used for
// authentication... It should complain saying container was recently
// stopped.
sb = new StringBuilder("Container ");
sb.Append(validContainerId);
sb.Append(" was recently stopped on node manager");
NUnit.Framework.Assert.IsTrue(TestGetContainer(rpc, validAppAttemptId, validNode,
validContainerId, validNMToken, true).Contains(sb.ToString()));
// Now lets remove the container from nm-memory
nm.GetNodeStatusUpdater().ClearFinishedContainersFromCache();
// This should fail as container is removed from recently tracked finished
// containers.
sb = new StringBuilder("Container ");
sb.Append(validContainerId.ToString());
sb.Append(" is not handled by this NodeManager");
NUnit.Framework.Assert.IsTrue(TestGetContainer(rpc, validAppAttemptId, validNode,
validContainerId, validNMToken, false).Contains(sb.ToString()));
// using appAttempt-1 NMtoken for launching container for appAttempt-2 should
// succeed.
ApplicationAttemptId attempt2 = ApplicationAttemptId.NewInstance(appId, 2);
Token attempt1NMToken = nmTokenSecretManagerRM.CreateNMToken(validAppAttemptId, validNode
, user);
Token newContainerToken = containerTokenSecretManager.CreateContainerToken(ContainerId
.NewContainerId(attempt2, 1), validNode, user, r, Priority.NewInstance(0), 0);
NUnit.Framework.Assert.IsTrue(TestStartContainer(rpc, attempt2, validNode, newContainerToken
, attempt1NMToken, false).IsEmpty());
}
/// <exception cref="System.IO.IOException"/>
public static FSDataInputStream GetPreviousJobHistoryFileStream(Configuration conf
, ApplicationAttemptId applicationAttemptId)
{
FSDataInputStream @in = null;
Path historyFile = null;
string jobId = TypeConverter.FromYarn(applicationAttemptId.GetApplicationId()).ToString
();
string jobhistoryDir = JobHistoryUtils.GetConfiguredHistoryStagingDirPrefix(conf,
jobId);
Path histDirPath = FileContext.GetFileContext(conf).MakeQualified(new Path(jobhistoryDir
));
FileContext fc = FileContext.GetFileContext(histDirPath.ToUri(), conf);
// read the previous history file
historyFile = fc.MakeQualified(JobHistoryUtils.GetStagingJobHistoryFile(histDirPath
, jobId, (applicationAttemptId.GetAttemptId() - 1)));
Log.Info("History file is at " + historyFile);
@in = fc.Open(historyFile);
return(@in);
}
