/// <exception cref="System.IO.IOException"/>
public override KeyProvider.KeyVersion CreateKey(string name, byte[] material, KeyProvider.Options
options)
{
lock (this)
{
Text nameT = new Text(name);
if (credentials.GetSecretKey(nameT) != null)
{
throw new IOException("Key " + name + " already exists in " + this);
}
if (options.GetBitLength() != 8 * material.Length)
{
throw new IOException("Wrong key length. Required " + options.GetBitLength() + ", but got "
+ (8 * material.Length));
}
KeyProvider.Metadata meta = new KeyProvider.Metadata(options.GetCipher(), options
.GetBitLength(), options.GetDescription(), options.GetAttributes(), new DateTime
(), 1);
cache[name] = meta;
string versionName = BuildVersionName(name, 0);
credentials.AddSecretKey(nameT, meta.Serialize());
credentials.AddSecretKey(new Text(versionName), material);
return(new KeyProvider.KeyVersion(name, versionName, material));
}
}
/// <exception cref="System.IO.IOException"/>
public override CredentialProvider.CredentialEntry CreateCredentialEntry(string name
, char[] credential)
{
Text nameT = new Text(name);
if (credentials.GetSecretKey(nameT) != null)
{
throw new IOException("Credential " + name + " already exists in " + this);
}
credentials.AddSecretKey(new Text(name), Runtime.GetBytesForString(new string
(credential), "UTF-8"));
return(new CredentialProvider.CredentialEntry(name, credential));
}
/// <exception cref="System.IO.IOException"/>
private void ReadTokensFromFiles(Configuration conf, Credentials credentials)
{
// add tokens and secrets coming from a token storage file
string binaryTokenFilename = conf.Get("mapreduce.job.credentials.binary");
if (binaryTokenFilename != null)
{
Credentials binary = Credentials.ReadTokenStorageFile(FileSystem.GetLocal(conf).MakeQualified
(new Path(binaryTokenFilename)), conf);
credentials.AddAll(binary);
}
// add secret keys coming from a json file
string tokensFileName = conf.Get("mapreduce.job.credentials.json");
if (tokensFileName != null)
{
Log.Info("loading user's secret keys from " + tokensFileName);
string localFileName = new Path(tokensFileName).ToUri().GetPath();
bool json_error = false;
try
{
// read JSON
ObjectMapper mapper = new ObjectMapper();
IDictionary <string, string> nm = mapper.ReadValue <IDictionary>(new FilePath(localFileName
));
foreach (KeyValuePair <string, string> ent in nm)
{
credentials.AddSecretKey(new Text(ent.Key), Sharpen.Runtime.GetBytesForString(ent
.Value, Charsets.Utf8));
}
}
catch (JsonMappingException)
{
json_error = true;
}
catch (JsonParseException)
{
json_error = true;
}
if (json_error)
{
Log.Warn("couldn't parse Token Cache JSON file with user secret keys");
}
}
}
public virtual void TestUGICredentialsPropogation()
{
Credentials creds = new Credentials();
Org.Apache.Hadoop.Security.Token.Token <object> token = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Text tokenService = new Text("service");
Text secretName = new Text("secret");
byte[] secret = new byte[] { };
creds.AddToken(tokenService, token);
creds.AddSecretKey(secretName, secret);
UserGroupInformation.GetLoginUser().AddCredentials(creds);
JobConf jobConf = new JobConf();
Job job = new Job(jobConf);
NUnit.Framework.Assert.AreSame(token, job.GetCredentials().GetToken(tokenService)
);
NUnit.Framework.Assert.AreSame(secret, job.GetCredentials().GetSecretKey(secretName
));
}
public virtual void TestMRAppMasterCredentials()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
// Simulate credentials passed to AM via client->RM->NM
Credentials credentials = new Credentials();
byte[] identifier = Sharpen.Runtime.GetBytesForString("MyIdentifier");
byte[] password = Sharpen.Runtime.GetBytesForString("MyPassword");
Text kind = new Text("MyTokenKind");
Text service = new Text("host:port");
Org.Apache.Hadoop.Security.Token.Token<TokenIdentifier> myToken = new Org.Apache.Hadoop.Security.Token.Token
<TokenIdentifier>(identifier, password, kind, service);
Text tokenAlias = new Text("myToken");
credentials.AddToken(tokenAlias, myToken);
Text appTokenService = new Text("localhost:0");
Org.Apache.Hadoop.Security.Token.Token<AMRMTokenIdentifier> appToken = new Org.Apache.Hadoop.Security.Token.Token
<AMRMTokenIdentifier>(identifier, password, AMRMTokenIdentifier.KindName, appTokenService
);
credentials.AddToken(appTokenService, appToken);
Text keyAlias = new Text("mySecretKeyAlias");
credentials.AddSecretKey(keyAlias, Sharpen.Runtime.GetBytesForString("mySecretKey"
));
Org.Apache.Hadoop.Security.Token.Token<TokenIdentifier> storedToken = credentials
.GetToken(tokenAlias);
JobConf conf = new JobConf();
Path tokenFilePath = new Path(testDir.GetAbsolutePath(), "tokens-file");
IDictionary<string, string> newEnv = new Dictionary<string, string>();
newEnv[UserGroupInformation.HadoopTokenFileLocation] = tokenFilePath.ToUri().GetPath
();
SetNewEnvironmentHack(newEnv);
credentials.WriteTokenStorageFile(tokenFilePath, conf);
ApplicationId appId = ApplicationId.NewInstance(12345, 56);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(appId
, 1);
ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 546);
string userName = UserGroupInformation.GetCurrentUser().GetShortUserName();
// Create staging dir, so MRAppMaster doesn't barf.
FilePath stagingDir = new FilePath(MRApps.GetStagingAreaDir(conf, userName).ToString
());
stagingDir.Mkdirs();
// Set login-user to null as that is how real world MRApp starts with.
// This is null is the reason why token-file is read by UGI.
UserGroupInformation.SetLoginUser(null);
MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId
, "host", -1, -1, Runtime.CurrentTimeMillis(), false, true);
MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName);
// Now validate the task credentials
Credentials appMasterCreds = appMaster.GetCredentials();
NUnit.Framework.Assert.IsNotNull(appMasterCreds);
NUnit.Framework.Assert.AreEqual(1, appMasterCreds.NumberOfSecretKeys());
NUnit.Framework.Assert.AreEqual(1, appMasterCreds.NumberOfTokens());
// Validate the tokens - app token should not be present
Org.Apache.Hadoop.Security.Token.Token<TokenIdentifier> usedToken = appMasterCreds
.GetToken(tokenAlias);
NUnit.Framework.Assert.IsNotNull(usedToken);
NUnit.Framework.Assert.AreEqual(storedToken, usedToken);
// Validate the keys
byte[] usedKey = appMasterCreds.GetSecretKey(keyAlias);
NUnit.Framework.Assert.IsNotNull(usedKey);
NUnit.Framework.Assert.AreEqual("mySecretKey", Sharpen.Runtime.GetStringForBytes(
usedKey));
// The credentials should also be added to conf so that OuputCommitter can
// access it - app token should not be present
Credentials confCredentials = conf.GetCredentials();
NUnit.Framework.Assert.AreEqual(1, confCredentials.NumberOfSecretKeys());
NUnit.Framework.Assert.AreEqual(1, confCredentials.NumberOfTokens());
NUnit.Framework.Assert.AreEqual(storedToken, confCredentials.GetToken(tokenAlias)
);
NUnit.Framework.Assert.AreEqual("mySecretKey", Sharpen.Runtime.GetStringForBytes(
confCredentials.GetSecretKey(keyAlias)));
// Verify the AM's ugi - app token should be present
Credentials ugiCredentials = appMaster.GetUgi().GetCredentials();
NUnit.Framework.Assert.AreEqual(1, ugiCredentials.NumberOfSecretKeys());
NUnit.Framework.Assert.AreEqual(2, ugiCredentials.NumberOfTokens());
NUnit.Framework.Assert.AreEqual(storedToken, ugiCredentials.GetToken(tokenAlias));
NUnit.Framework.Assert.AreEqual(appToken, ugiCredentials.GetToken(appTokenService
));
NUnit.Framework.Assert.AreEqual("mySecretKey", Sharpen.Runtime.GetStringForBytes(
ugiCredentials.GetSecretKey(keyAlias)));
}
public static void SetEncryptedSpillKey(byte[] key, Credentials credentials)
{
credentials.AddSecretKey(EncSpillKey, key);
}
public static void SetShuffleSecretKey(byte[] key, Credentials credentials)
{
credentials.AddSecretKey(ShuffleToken, key);
}
public virtual void TestAttemptContainerRequest()
{
Text SecretKeyAlias = new Text("secretkeyalias");
byte[] SecretKey = Sharpen.Runtime.GetBytesForString(("secretkey"));
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>(1);
acls[ApplicationAccessType.ViewApp] = "otheruser";
ApplicationId appId = ApplicationId.NewInstance(1, 1);
JobId jobId = MRBuilderUtils.NewJobId(appId, 1);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Map);
Path jobFile = Org.Mockito.Mockito.Mock <Path>();
EventHandler eventHandler = Org.Mockito.Mockito.Mock <EventHandler>();
TaskAttemptListener taListener = Org.Mockito.Mockito.Mock <TaskAttemptListener>();
Org.Mockito.Mockito.When(taListener.GetAddress()).ThenReturn(new IPEndPoint("localhost"
, 0));
JobConf jobConf = new JobConf();
jobConf.SetClass("fs.file.impl", typeof(TestTaskAttemptContainerRequest.StubbedFS
), typeof(FileSystem));
jobConf.SetBoolean("fs.file.impl.disable.cache", true);
jobConf.Set(JobConf.MapredMapTaskEnv, string.Empty);
// setup UGI for security so tokens and keys are preserved
jobConf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"
);
UserGroupInformation.SetConfiguration(jobConf);
Credentials credentials = new Credentials();
credentials.AddSecretKey(SecretKeyAlias, SecretKey);
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jobToken = new Org.Apache.Hadoop.Security.Token.Token
<JobTokenIdentifier>(Sharpen.Runtime.GetBytesForString(("tokenid")), Sharpen.Runtime.GetBytesForString
(("tokenpw")), new Text("tokenkind"), new Text("tokenservice"));
TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile,
1, Org.Mockito.Mockito.Mock <JobSplit.TaskSplitMetaInfo>(), jobConf, taListener,
jobToken, credentials, new SystemClock(), null);
jobConf.Set(MRJobConfig.ApplicationAttemptId, taImpl.GetID().ToString());
ContainerLaunchContext launchCtx = TaskAttemptImpl.CreateContainerLaunchContext(acls
, jobConf, jobToken, taImpl.CreateRemoteTask(), TypeConverter.FromYarn(jobId), Org.Mockito.Mockito.Mock
<WrappedJvmID>(), taListener, credentials);
NUnit.Framework.Assert.AreEqual("ACLs mismatch", acls, launchCtx.GetApplicationACLs
());
Credentials launchCredentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
dibb.Reset(launchCtx.GetTokens());
launchCredentials.ReadTokenStorageStream(dibb);
// verify all tokens specified for the task attempt are in the launch context
foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> token in credentials
.GetAllTokens())
{
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> launchToken = launchCredentials
.GetToken(token.GetService());
NUnit.Framework.Assert.IsNotNull("Token " + token.GetService() + " is missing", launchToken
);
NUnit.Framework.Assert.AreEqual("Token " + token.GetService() + " mismatch", token
, launchToken);
}
// verify the secret key is in the launch context
NUnit.Framework.Assert.IsNotNull("Secret key missing", launchCredentials.GetSecretKey
(SecretKeyAlias));
NUnit.Framework.Assert.IsTrue("Secret key mismatch", Arrays.Equals(SecretKey, launchCredentials
.GetSecretKey(SecretKeyAlias)));
}
