public virtual void TestShuffleProviders()
{
ApplicationId appId = ApplicationId.NewInstance(1, 1);
JobId jobId = MRBuilderUtils.NewJobId(appId, 1);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Map);
Path jobFile = Org.Mockito.Mockito.Mock <Path>();
EventHandler eventHandler = Org.Mockito.Mockito.Mock <EventHandler>();
TaskAttemptListener taListener = Org.Mockito.Mockito.Mock <TaskAttemptListener>();
Org.Mockito.Mockito.When(taListener.GetAddress()).ThenReturn(new IPEndPoint("localhost"
, 0));
JobConf jobConf = new JobConf();
jobConf.SetClass("fs.file.impl", typeof(TestShuffleProvider.StubbedFS), typeof(FileSystem
));
jobConf.SetBoolean("fs.file.impl.disable.cache", true);
jobConf.Set(JobConf.MapredMapTaskEnv, string.Empty);
jobConf.Set(YarnConfiguration.NmAuxServices, TestShuffleProvider.TestShuffleHandler1
.MapreduceTestShuffleServiceid + "," + TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid
);
string serviceName = TestShuffleProvider.TestShuffleHandler1.MapreduceTestShuffleServiceid;
string serviceStr = string.Format(YarnConfiguration.NmAuxServiceFmt, serviceName);
jobConf.Set(serviceStr, typeof(TestShuffleProvider.TestShuffleHandler1).FullName);
serviceName = TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid;
serviceStr = string.Format(YarnConfiguration.NmAuxServiceFmt, serviceName);
jobConf.Set(serviceStr, typeof(TestShuffleProvider.TestShuffleHandler2).FullName);
jobConf.Set(MRJobConfig.MapreduceJobShuffleProviderServices, TestShuffleProvider.TestShuffleHandler1
.MapreduceTestShuffleServiceid + "," + TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid
);
Credentials credentials = new Credentials();
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jobToken = new Org.Apache.Hadoop.Security.Token.Token
<JobTokenIdentifier>(Sharpen.Runtime.GetBytesForString(("tokenid")), Sharpen.Runtime.GetBytesForString
(("tokenpw")), new Text("tokenkind"), new Text("tokenservice"));
TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile,
1, Org.Mockito.Mockito.Mock <JobSplit.TaskSplitMetaInfo>(), jobConf, taListener,
jobToken, credentials, new SystemClock(), null);
jobConf.Set(MRJobConfig.ApplicationAttemptId, taImpl.GetID().ToString());
ContainerLaunchContext launchCtx = TaskAttemptImpl.CreateContainerLaunchContext(null
, jobConf, jobToken, taImpl.CreateRemoteTask(), TypeConverter.FromYarn(jobId), Org.Mockito.Mockito.Mock
<WrappedJvmID>(), taListener, credentials);
IDictionary <string, ByteBuffer> serviceDataMap = launchCtx.GetServiceData();
NUnit.Framework.Assert.IsNotNull("TestShuffleHandler1 is missing", serviceDataMap
[TestShuffleProvider.TestShuffleHandler1.MapreduceTestShuffleServiceid]);
NUnit.Framework.Assert.IsNotNull("TestShuffleHandler2 is missing", serviceDataMap
[TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid]);
NUnit.Framework.Assert.IsTrue("mismatch number of services in map", serviceDataMap
.Count == 3);
}
public virtual void TestAttemptContainerRequest()
{
Text SecretKeyAlias = new Text("secretkeyalias");
byte[] SecretKey = Sharpen.Runtime.GetBytesForString(("secretkey"));
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>(1);
acls[ApplicationAccessType.ViewApp] = "otheruser";
ApplicationId appId = ApplicationId.NewInstance(1, 1);
JobId jobId = MRBuilderUtils.NewJobId(appId, 1);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Map);
Path jobFile = Org.Mockito.Mockito.Mock <Path>();
EventHandler eventHandler = Org.Mockito.Mockito.Mock <EventHandler>();
TaskAttemptListener taListener = Org.Mockito.Mockito.Mock <TaskAttemptListener>();
Org.Mockito.Mockito.When(taListener.GetAddress()).ThenReturn(new IPEndPoint("localhost"
, 0));
JobConf jobConf = new JobConf();
jobConf.SetClass("fs.file.impl", typeof(TestTaskAttemptContainerRequest.StubbedFS
), typeof(FileSystem));
jobConf.SetBoolean("fs.file.impl.disable.cache", true);
jobConf.Set(JobConf.MapredMapTaskEnv, string.Empty);
// setup UGI for security so tokens and keys are preserved
jobConf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"
);
UserGroupInformation.SetConfiguration(jobConf);
Credentials credentials = new Credentials();
credentials.AddSecretKey(SecretKeyAlias, SecretKey);
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jobToken = new Org.Apache.Hadoop.Security.Token.Token
<JobTokenIdentifier>(Sharpen.Runtime.GetBytesForString(("tokenid")), Sharpen.Runtime.GetBytesForString
(("tokenpw")), new Text("tokenkind"), new Text("tokenservice"));
TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile,
1, Org.Mockito.Mockito.Mock <JobSplit.TaskSplitMetaInfo>(), jobConf, taListener,
jobToken, credentials, new SystemClock(), null);
jobConf.Set(MRJobConfig.ApplicationAttemptId, taImpl.GetID().ToString());
ContainerLaunchContext launchCtx = TaskAttemptImpl.CreateContainerLaunchContext(acls
, jobConf, jobToken, taImpl.CreateRemoteTask(), TypeConverter.FromYarn(jobId), Org.Mockito.Mockito.Mock
<WrappedJvmID>(), taListener, credentials);
NUnit.Framework.Assert.AreEqual("ACLs mismatch", acls, launchCtx.GetApplicationACLs
());
Credentials launchCredentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
dibb.Reset(launchCtx.GetTokens());
launchCredentials.ReadTokenStorageStream(dibb);
// verify all tokens specified for the task attempt are in the launch context
foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> token in credentials
.GetAllTokens())
{
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> launchToken = launchCredentials
.GetToken(token.GetService());
NUnit.Framework.Assert.IsNotNull("Token " + token.GetService() + " is missing", launchToken
);
NUnit.Framework.Assert.AreEqual("Token " + token.GetService() + " mismatch", token
, launchToken);
}
// verify the secret key is in the launch context
NUnit.Framework.Assert.IsNotNull("Secret key missing", launchCredentials.GetSecretKey
(SecretKeyAlias));
NUnit.Framework.Assert.IsTrue("Secret key mismatch", Arrays.Equals(SecretKey, launchCredentials
.GetSecretKey(SecretKeyAlias)));
}