/// <exception cref="System.IO.IOException"/>
private static IDictionary <ApplicationId, Credentials> ParseCredentials(IDictionary
<ApplicationId, ByteBuffer> systemCredentials)
{
IDictionary <ApplicationId, Credentials> map = new Dictionary <ApplicationId, Credentials
>();
foreach (KeyValuePair <ApplicationId, ByteBuffer> entry in systemCredentials)
{
Credentials credentials = new Credentials();
DataInputByteBuffer buf = new DataInputByteBuffer();
ByteBuffer buffer = entry.Value;
buffer.Rewind();
buf.Reset(buffer);
credentials.ReadTokenStorageStream(buf);
map[entry.Key] = credentials;
}
if (Log.IsDebugEnabled())
{
foreach (KeyValuePair <ApplicationId, Credentials> entry_1 in map)
{
Log.Debug("Retrieved credentials form RM for " + entry_1.Key + ": " + entry_1.Value
.GetAllTokens());
}
}
return(map);
}
private static Credentials ConvertCredentialsFromByteBuffer(ByteBuffer appAttemptTokens
)
{
DataInputByteBuffer dibb = new DataInputByteBuffer();
try
{
Credentials credentials = null;
if (appAttemptTokens != null)
{
credentials = new Credentials();
appAttemptTokens.Rewind();
dibb.Reset(appAttemptTokens);
credentials.ReadTokenStorageStream(dibb);
}
return(credentials);
}
catch (IOException)
{
Log.Error("Failed to convert Credentials from ByteBuffer.");
System.Diagnostics.Debug.Assert(false);
return(null);
}
finally
{
IOUtils.CloseStream(dibb);
}
}
/// <exception cref="System.IO.IOException"/>
private void SetupTokens(ContainerLaunchContext container, ContainerId containerID
)
{
IDictionary <string, string> environment = container.GetEnvironment();
environment[ApplicationConstants.ApplicationWebProxyBaseEnv] = application.GetWebProxyBase
();
// Set AppSubmitTime and MaxAppAttempts to be consumable by the AM.
ApplicationId applicationId = application.GetAppAttemptId().GetApplicationId();
environment[ApplicationConstants.AppSubmitTimeEnv] = rmContext.GetRMApps()[applicationId
].GetSubmitTime().ToString();
environment[ApplicationConstants.MaxAppAttemptsEnv] = rmContext.GetRMApps()[applicationId
].GetMaxAppAttempts().ToString();
Credentials credentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
if (container.GetTokens() != null)
{
// TODO: Don't do this kind of checks everywhere.
dibb.Reset(container.GetTokens());
credentials.ReadTokenStorageStream(dibb);
}
// Add AMRMToken
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = CreateAndSetAMRMToken
();
if (amrmToken != null)
{
credentials.AddToken(amrmToken.GetService(), amrmToken);
}
DataOutputBuffer dob = new DataOutputBuffer();
credentials.WriteTokenStorageToStream(dob);
container.SetTokens(ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength()));
}
/// <summary>A helper function to deserialize the metadata returned by ShuffleHandler.
/// </summary>
/// <param name="meta">the metadata returned by the ShuffleHandler</param>
/// <returns>the port the Shuffle Handler is listening on to serve shuffle data.</returns>
/// <exception cref="System.IO.IOException"/>
public static int DeserializeMetaData(ByteBuffer meta)
{
//TODO this should be returning a class not just an int
DataInputByteBuffer @in = new DataInputByteBuffer();
@in.Reset(meta);
int port = @in.ReadInt();
return port;
}
public virtual void TestByteBuffers()
{
DataOutputByteBuffer dob = new DataOutputByteBuffer();
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
System.Console.Out.WriteLine("SEED: " + seed);
WriteJunk(dob, r, seed, 1000);
DataInputByteBuffer dib = new DataInputByteBuffer();
dib.Reset(dob.GetData());
ReadJunk(dib, r, seed, 1000);
dob.Reset();
WriteJunk(dob, r, seed, 1000);
dib.Reset(dob.GetData());
ReadJunk(dib, r, seed, 1000);
}
/// <exception cref="System.IO.IOException"/>
internal static Org.Apache.Hadoop.Security.Token.Token<JobTokenIdentifier> DeserializeServiceData
(ByteBuffer secret)
{
DataInputByteBuffer @in = new DataInputByteBuffer();
@in.Reset(secret);
Org.Apache.Hadoop.Security.Token.Token<JobTokenIdentifier> jt = new Org.Apache.Hadoop.Security.Token.Token
<JobTokenIdentifier>();
jt.ReadFields(@in);
return jt;
}
/// <exception cref="System.IO.IOException"/>
public Credentials GetContainerCredentials()
{
Credentials credentials = new Credentials();
DataInputByteBuffer buf = new DataInputByteBuffer();
containerTokens.Rewind();
buf.Reset(containerTokens);
credentials.ReadTokenStorageStream(buf);
return(credentials);
}
public virtual void TestDataInputByteBufferCompatibility()
{
DataOutputBuffer dob = new DataOutputBuffer();
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
System.Console.Out.WriteLine("SEED: " + seed);
WriteJunk(dob, r, seed, 1000);
ByteBuffer buf = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
DataInputByteBuffer dib = new DataInputByteBuffer();
dib.Reset(buf);
ReadJunk(dib, r, seed, 1000);
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual Credentials ParseCredentials(ApplicationSubmissionContext
application)
{
Credentials credentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
ByteBuffer tokens = application.GetAMContainerSpec().GetTokens();
if (tokens != null)
{
dibb.Reset(tokens);
credentials.ReadTokenStorageStream(dibb);
tokens.Rewind();
}
return(credentials);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private void AddTimelineDelegationToken(ContainerLaunchContext clc)
{
Credentials credentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
ByteBuffer tokens = clc.GetTokens();
if (tokens != null)
{
dibb.Reset(tokens);
credentials.ReadTokenStorageStream(dibb);
tokens.Rewind();
}
// If the timeline delegation token is already in the CLC, no need to add
// one more
foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> token in credentials
.GetAllTokens())
{
if (token.GetKind().Equals(TimelineDelegationTokenIdentifier.KindName))
{
return;
}
}
Org.Apache.Hadoop.Security.Token.Token <TimelineDelegationTokenIdentifier> timelineDelegationToken
= GetTimelineDelegationToken();
if (timelineDelegationToken == null)
{
return;
}
credentials.AddToken(timelineService, timelineDelegationToken);
if (Log.IsDebugEnabled())
{
Log.Debug("Add timline delegation token into credentials: " + timelineDelegationToken
);
}
DataOutputBuffer dob = new DataOutputBuffer();
credentials.WriteTokenStorageToStream(dob);
tokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
clc.SetTokens(tokens);
}
public virtual void TestAttemptContainerRequest()
{
Text SecretKeyAlias = new Text("secretkeyalias");
byte[] SecretKey = Sharpen.Runtime.GetBytesForString(("secretkey"));
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>(1);
acls[ApplicationAccessType.ViewApp] = "otheruser";
ApplicationId appId = ApplicationId.NewInstance(1, 1);
JobId jobId = MRBuilderUtils.NewJobId(appId, 1);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Map);
Path jobFile = Org.Mockito.Mockito.Mock <Path>();
EventHandler eventHandler = Org.Mockito.Mockito.Mock <EventHandler>();
TaskAttemptListener taListener = Org.Mockito.Mockito.Mock <TaskAttemptListener>();
Org.Mockito.Mockito.When(taListener.GetAddress()).ThenReturn(new IPEndPoint("localhost"
, 0));
JobConf jobConf = new JobConf();
jobConf.SetClass("fs.file.impl", typeof(TestTaskAttemptContainerRequest.StubbedFS
), typeof(FileSystem));
jobConf.SetBoolean("fs.file.impl.disable.cache", true);
jobConf.Set(JobConf.MapredMapTaskEnv, string.Empty);
// setup UGI for security so tokens and keys are preserved
jobConf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"
);
UserGroupInformation.SetConfiguration(jobConf);
Credentials credentials = new Credentials();
credentials.AddSecretKey(SecretKeyAlias, SecretKey);
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jobToken = new Org.Apache.Hadoop.Security.Token.Token
<JobTokenIdentifier>(Sharpen.Runtime.GetBytesForString(("tokenid")), Sharpen.Runtime.GetBytesForString
(("tokenpw")), new Text("tokenkind"), new Text("tokenservice"));
TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile,
1, Org.Mockito.Mockito.Mock <JobSplit.TaskSplitMetaInfo>(), jobConf, taListener,
jobToken, credentials, new SystemClock(), null);
jobConf.Set(MRJobConfig.ApplicationAttemptId, taImpl.GetID().ToString());
ContainerLaunchContext launchCtx = TaskAttemptImpl.CreateContainerLaunchContext(acls
, jobConf, jobToken, taImpl.CreateRemoteTask(), TypeConverter.FromYarn(jobId), Org.Mockito.Mockito.Mock
<WrappedJvmID>(), taListener, credentials);
NUnit.Framework.Assert.AreEqual("ACLs mismatch", acls, launchCtx.GetApplicationACLs
());
Credentials launchCredentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
dibb.Reset(launchCtx.GetTokens());
launchCredentials.ReadTokenStorageStream(dibb);
// verify all tokens specified for the task attempt are in the launch context
foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> token in credentials
.GetAllTokens())
{
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> launchToken = launchCredentials
.GetToken(token.GetService());
NUnit.Framework.Assert.IsNotNull("Token " + token.GetService() + " is missing", launchToken
);
NUnit.Framework.Assert.AreEqual("Token " + token.GetService() + " mismatch", token
, launchToken);
}
// verify the secret key is in the launch context
NUnit.Framework.Assert.IsNotNull("Secret key missing", launchCredentials.GetSecretKey
(SecretKeyAlias));
NUnit.Framework.Assert.IsTrue("Secret key mismatch", Arrays.Equals(SecretKey, launchCredentials
.GetSecretKey(SecretKeyAlias)));
}