/// <summary>
/// Download <code>fsimage</code> and <code>edits</code>
/// files from the name-node.
/// </summary>
/// <returns>true if a new image has been downloaded and needs to be loaded</returns>
/// <exception cref="System.IO.IOException"/>
internal static bool DownloadCheckpointFiles(Uri nnHostPort, FSImage dstImage, CheckpointSignature
sig, RemoteEditLogManifest manifest)
{
// Sanity check manifest - these could happen if, eg, someone on the
// NN side accidentally rmed the storage directories
if (manifest.GetLogs().IsEmpty())
{
throw new IOException("Found no edit logs to download on NN since txid " + sig.mostRecentCheckpointTxId
);
}
long expectedTxId = sig.mostRecentCheckpointTxId + 1;
if (manifest.GetLogs()[0].GetStartTxId() != expectedTxId)
{
throw new IOException("Bad edit log manifest (expected txid = " + expectedTxId +
": " + manifest);
}
try
{
bool b = UserGroupInformation.GetCurrentUser().DoAs(new _PrivilegedExceptionAction_444
(dstImage, sig, nnHostPort, manifest));
// get fsimage
// get edits file
// true if we haven't loaded all the transactions represented by the
// downloaded fsimage.
return(b);
}
catch (Exception e)
{
throw new RuntimeException(e);
}
}
public virtual void TestJobTokenRpc()
{
TaskUmbilicalProtocol mockTT = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol>();
Org.Mockito.Mockito.DoReturn(TaskUmbilicalProtocol.versionID).When(mockTT).GetProtocolVersion
(Matchers.AnyString(), Matchers.AnyLong());
Org.Mockito.Mockito.DoReturn(ProtocolSignature.GetProtocolSignature(mockTT, typeof(
TaskUmbilicalProtocol).FullName, TaskUmbilicalProtocol.versionID, 0)).When(mockTT
).GetProtocolSignature(Matchers.AnyString(), Matchers.AnyLong(), Matchers.AnyInt
());
JobTokenSecretManager sm = new JobTokenSecretManager();
Server server = new RPC.Builder(conf).SetProtocol(typeof(TaskUmbilicalProtocol)).
SetInstance(mockTT).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose
(true).SetSecretManager(sm).Build();
server.Start();
UserGroupInformation current = UserGroupInformation.GetCurrentUser();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
string jobId = current.GetUserName();
JobTokenIdentifier tokenId = new JobTokenIdentifier(new Text(jobId));
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<JobTokenIdentifier>(tokenId, sm);
sm.AddTokenForJob(jobId, token);
SecurityUtil.SetTokenService(token, addr);
Log.Info("Service address for token is " + token.GetService());
current.AddToken(token);
current.DoAs(new _PrivilegedExceptionAction_110(addr, server));
}
// There are two scenarios when RM failover happens
// during SubmitApplication Call:
// 1) RMStateStore already saved the ApplicationState when failover happens
// 2) RMStateStore did not save the ApplicationState when failover happens
/// <exception cref="System.Exception"/>
public virtual void TestHandleRMHADuringSubmitApplicationCallWithSavedApplicationState
()
{
// Test scenario 1 when RM failover happens
// druing SubmitApplication Call:
// RMStateStore already saved the ApplicationState when failover happens
StartRMs();
// Submit Application
// After submission, the applicationState will be saved in RMStateStore.
RMApp app0 = rm1.SubmitApp(200);
// Do the failover
ExplicitFailover();
// Since the applicationState has already been saved in RMStateStore
// before failover happens, the current active rm can load the previous
// applicationState.
// This RMApp should exist in the RMContext of current active RM
NUnit.Framework.Assert.IsTrue(rm2.GetRMContext().GetRMApps().Contains(app0.GetApplicationId
()));
// When we re-submit the application with same applicationId, it will
// check whether this application has been exist. If yes, just simply
// return submitApplicationResponse.
RMApp app1 = rm2.SubmitApp(200, string.Empty, UserGroupInformation.GetCurrentUser
().GetShortUserName(), null, false, null, configuration.GetInt(YarnConfiguration
.RmAmMaxAttempts, YarnConfiguration.DefaultRmAmMaxAttempts), null, null, false,
false, true, app0.GetApplicationId());
NUnit.Framework.Assert.AreEqual(app1.GetApplicationId(), app0.GetApplicationId());
}
/// <exception cref="System.Exception"/>
public virtual void TestUGIAuthMethodInRealUser()
{
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
UserGroupInformation proxyUgi = UserGroupInformation.CreateProxyUser("proxy", ugi
);
UserGroupInformation.AuthenticationMethod am = UserGroupInformation.AuthenticationMethod
.Kerberos;
ugi.SetAuthenticationMethod(am);
Assert.Equal(am, ugi.GetAuthenticationMethod());
Assert.Equal(UserGroupInformation.AuthenticationMethod.Proxy,
proxyUgi.GetAuthenticationMethod());
Assert.Equal(am, UserGroupInformation.GetRealAuthenticationMethod
(proxyUgi));
proxyUgi.DoAs(new _PrivilegedExceptionAction_690(am));
UserGroupInformation proxyUgi2 = new UserGroupInformation(proxyUgi.GetSubject());
proxyUgi2.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Proxy
);
Assert.Equal(proxyUgi, proxyUgi2);
// Equality should work if authMethod is null
UserGroupInformation realugi = UserGroupInformation.GetCurrentUser();
UserGroupInformation proxyUgi3 = UserGroupInformation.CreateProxyUser("proxyAnother"
, realugi);
UserGroupInformation proxyUgi4 = new UserGroupInformation(proxyUgi3.GetSubject());
Assert.Equal(proxyUgi3, proxyUgi4);
}
/// <exception cref="System.IO.IOException"/>
public virtual GetDelegationTokenResponse GetDelegationToken(GetDelegationTokenRequest
request)
{
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
// Verify that the connection is kerberos authenticated
if (!this.IsAllowedDelegationTokenOp())
{
throw new IOException("Delegation Token can be issued only with kerberos authentication"
);
}
GetDelegationTokenResponse response = this.recordFactory.NewRecordInstance <GetDelegationTokenResponse
>();
string user = ugi.GetUserName();
Text owner = new Text(user);
Text realUser = null;
if (ugi.GetRealUser() != null)
{
realUser = new Text(ugi.GetRealUser().GetUserName());
}
MRDelegationTokenIdentifier tokenIdentifier = new MRDelegationTokenIdentifier(owner
, new Text(request.GetRenewer()), realUser);
Org.Apache.Hadoop.Security.Token.Token <MRDelegationTokenIdentifier> realJHSToken =
new Org.Apache.Hadoop.Security.Token.Token <MRDelegationTokenIdentifier>(tokenIdentifier
, this._enclosing.jhsDTSecretManager);
Org.Apache.Hadoop.Yarn.Api.Records.Token mrDToken = Org.Apache.Hadoop.Yarn.Api.Records.Token
.NewInstance(realJHSToken.GetIdentifier(), realJHSToken.GetKind().ToString(), realJHSToken
.GetPassword(), realJHSToken.GetService().ToString());
response.SetDelegationToken(mrDToken);
return(response);
}
/// <exception cref="System.Exception"/>
public virtual void TestUniqueDestinationPath()
{
Configuration conf = new Configuration();
FileContext files = FileContext.GetLocalFSFileContext(conf);
Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name
));
files.Mkdir(basedir, null, true);
conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString());
ExecutorService singleThreadedExec = Executors.NewSingleThreadExecutor();
LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName);
Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), conf);
destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
()));
Path p = new Path(basedir, "dir" + 0 + ".jar");
LocalResourceVisibility vis = LocalResourceVisibility.Private;
LocalResource rsrc = CreateJar(files, p, vis);
FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf
, destPath, rsrc);
Future <Path> rPath = singleThreadedExec.Submit(fsd);
singleThreadedExec.Shutdown();
while (!singleThreadedExec.AwaitTermination(1000, TimeUnit.Milliseconds))
{
}
NUnit.Framework.Assert.IsTrue(rPath.IsDone());
// Now FSDownload will not create a random directory to localize the
// resource. Therefore the final localizedPath for the resource should be
// destination directory (passed as an argument) + file name.
NUnit.Framework.Assert.AreEqual(destPath, rPath.Get().GetParent());
}
/// <exception cref="Javax.Servlet.ServletException"/>
/// <exception cref="System.IO.IOException"/>
protected override void DoPut(HttpServletRequest request, HttpServletResponse response
)
{
try
{
ServletContext context = GetServletContext();
FSImage nnImage = NameNodeHttpServer.GetFsImageFromContext(context);
Configuration conf = (Configuration)GetServletContext().GetAttribute(JspHelper.CurrentConf
);
ImageServlet.PutImageParams parsedParams = new ImageServlet.PutImageParams(request
, response, conf);
NameNodeMetrics metrics = NameNode.GetNameNodeMetrics();
ValidateRequest(context, conf, request, response, nnImage, parsedParams.GetStorageInfoString
());
UserGroupInformation.GetCurrentUser().DoAs(new _PrivilegedExceptionAction_458(parsedParams
, nnImage, response, request, conf, metrics));
}
catch (Exception t)
{
// Metrics non-null only when used inside name node
// Now that we have a new checkpoint, we might be able to
// remove some old ones.
string errMsg = "PutImage failed. " + StringUtils.StringifyException(t);
response.SendError(HttpServletResponse.ScGone, errMsg);
throw new IOException(errMsg);
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual UseSharedCacheResourceResponse Use(UseSharedCacheResourceRequest request
)
{
UseSharedCacheResourceResponse response = recordFactory.NewRecordInstance <UseSharedCacheResourceResponse
>();
UserGroupInformation callerUGI;
try
{
callerUGI = UserGroupInformation.GetCurrentUser();
}
catch (IOException ie)
{
Log.Info("Error getting UGI ", ie);
throw RPCUtil.GetRemoteException(ie);
}
string fileName = this.store.AddResourceReference(request.GetResourceKey(), new SharedCacheResourceReference
(request.GetAppId(), callerUGI.GetShortUserName()));
if (fileName != null)
{
response.SetPath(GetCacheEntryFilePath(request.GetResourceKey(), fileName));
this.metrics.IncCacheHitCount();
}
else
{
this.metrics.IncCacheMissCount();
}
return(response);
}
/// <summary>Initializes the staging directory and returns the path.</summary>
/// <remarks>
/// Initializes the staging directory and returns the path. It also
/// keeps track of all necessary ownership and permissions
/// </remarks>
/// <param name="cluster"/>
/// <param name="conf"/>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public static Path GetStagingDir(Cluster cluster, Configuration conf)
{
Path stagingArea = cluster.GetStagingAreaDir();
FileSystem fs = stagingArea.GetFileSystem(conf);
string realUser;
string currentUser;
UserGroupInformation ugi = UserGroupInformation.GetLoginUser();
realUser = ugi.GetShortUserName();
currentUser = UserGroupInformation.GetCurrentUser().GetShortUserName();
if (fs.Exists(stagingArea))
{
FileStatus fsStatus = fs.GetFileStatus(stagingArea);
string owner = fsStatus.GetOwner();
if (!(owner.Equals(currentUser) || owner.Equals(realUser)))
{
throw new IOException("The ownership on the staging directory " + stagingArea + " is not as expected. "
+ "It is owned by " + owner + ". The directory must " + "be owned by the submitter "
+ currentUser + " or " + "by " + realUser);
}
if (!fsStatus.GetPermission().Equals(JobDirPermission))
{
Log.Info("Permissions on staging directory " + stagingArea + " are " + "incorrect: "
+ fsStatus.GetPermission() + ". Fixing permissions " + "to correct value " + JobDirPermission
);
fs.SetPermission(stagingArea, JobDirPermission);
}
}
else
{
fs.Mkdirs(stagingArea, new FsPermission(JobDirPermission));
}
return(stagingArea);
}
/// <exception cref="System.IO.IOException"/>
private HttpURLConnection CreateConnection(Uri url, string method)
{
HttpURLConnection conn;
try
{
// if current UGI is different from UGI at constructor time, behave as
// proxyuser
UserGroupInformation currentUgi = UserGroupInformation.GetCurrentUser();
string doAsUser = (currentUgi.GetAuthenticationMethod() == UserGroupInformation.AuthenticationMethod
.Proxy) ? currentUgi.GetShortUserName() : null;
// creating the HTTP connection using the current UGI at constructor time
conn = actualUgi.DoAs(new _PrivilegedExceptionAction_478(this, url, doAsUser));
}
catch (IOException ex)
{
throw;
}
catch (UndeclaredThrowableException ex)
{
throw new IOException(ex.GetUndeclaredThrowable());
}
catch (Exception ex)
{
throw new IOException(ex);
}
conn.SetUseCaches(false);
conn.SetRequestMethod(method);
if (method.Equals(HttpPost) || method.Equals(HttpPut))
{
conn.SetDoOutput(true);
}
conn = ConfigureConnection(conn);
return(conn);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual ReleaseSharedCacheResourceResponse Release(ReleaseSharedCacheResourceRequest
request)
{
ReleaseSharedCacheResourceResponse response = recordFactory.NewRecordInstance <ReleaseSharedCacheResourceResponse
>();
UserGroupInformation callerUGI;
try
{
callerUGI = UserGroupInformation.GetCurrentUser();
}
catch (IOException ie)
{
Log.Info("Error getting UGI ", ie);
throw RPCUtil.GetRemoteException(ie);
}
bool removed = this.store.RemoveResourceReference(request.GetResourceKey(), new SharedCacheResourceReference
(request.GetAppId(), callerUGI.GetShortUserName()), true);
if (removed)
{
this.metrics.IncCacheRelease();
}
return(response);
}
/// <exception cref="System.IO.IOException"/>
public KMSClientProvider(URI uri, Configuration conf)
: base(conf)
{
kmsUrl = CreateServiceURL(ExtractKMSPath(uri));
if (Runtime.EqualsIgnoreCase("https", new Uri(kmsUrl).Scheme))
{
sslFactory = new SSLFactory(SSLFactory.Mode.Client, conf);
try
{
sslFactory.Init();
}
catch (GeneralSecurityException ex)
{
throw new IOException(ex);
}
}
int timeout = conf.GetInt(TimeoutAttr, DefaultTimeout);
authRetry = conf.GetInt(AuthRetry, DefaultAuthRetry);
configurator = new KMSClientProvider.TimeoutConnConfigurator(timeout, sslFactory);
encKeyVersionQueue = new ValueQueue <KeyProviderCryptoExtension.EncryptedKeyVersion
>(conf.GetInt(CommonConfigurationKeysPublic.KmsClientEncKeyCacheSize, CommonConfigurationKeysPublic
.KmsClientEncKeyCacheSizeDefault), conf.GetFloat(CommonConfigurationKeysPublic.KmsClientEncKeyCacheLowWatermark
, CommonConfigurationKeysPublic.KmsClientEncKeyCacheLowWatermarkDefault), conf.GetInt
(CommonConfigurationKeysPublic.KmsClientEncKeyCacheExpiryMs, CommonConfigurationKeysPublic
.KmsClientEncKeyCacheExpiryDefault), conf.GetInt(CommonConfigurationKeysPublic.KmsClientEncKeyCacheNumRefillThreads
, CommonConfigurationKeysPublic.KmsClientEncKeyCacheNumRefillThreadsDefault), new
KMSClientProvider.EncryptedQueueRefiller(this));
authToken = new DelegationTokenAuthenticatedURL.Token();
actualUgi = (UserGroupInformation.GetCurrentUser().GetAuthenticationMethod() == UserGroupInformation.AuthenticationMethod
.Proxy) ? UserGroupInformation.GetCurrentUser().GetRealUser() : UserGroupInformation
.GetCurrentUser();
}
/// <exception cref="System.Exception"/>
private void WriteLog(Configuration configuration, string user)
{
ApplicationId appId = ApplicationIdPBImpl.NewInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.NewInstance(appId,
1);
ContainerId containerId = ContainerIdPBImpl.NewContainerId(appAttemptId, 1);
string path = "target/logs/" + user + "/logs/application_0_0001/localhost_1234";
FilePath f = new FilePath(path);
if (!f.GetParentFile().Exists())
{
NUnit.Framework.Assert.IsTrue(f.GetParentFile().Mkdirs());
}
IList <string> rootLogDirs = Arrays.AsList("target/logs/logs");
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
AggregatedLogFormat.LogWriter writer = new AggregatedLogFormat.LogWriter(configuration
, new Path(path), ugi);
writer.WriteApplicationOwner(ugi.GetUserName());
IDictionary <ApplicationAccessType, string> appAcls = new Dictionary <ApplicationAccessType
, string>();
appAcls[ApplicationAccessType.ViewApp] = ugi.GetUserName();
writer.WriteApplicationACLs(appAcls);
writer.Append(new AggregatedLogFormat.LogKey("container_0_0001_01_000001"), new AggregatedLogFormat.LogValue
(rootLogDirs, containerId, UserGroupInformation.GetCurrentUser().GetShortUserName
()));
writer.Close();
}
/// <exception cref="System.IO.IOException"/>
private void DisplayQueueAclsInfoForCurrentUser()
{
QueueAclsInfo[] queueAclsInfoList = jc.GetQueueAclsForCurrentUser();
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
if (queueAclsInfoList.Length > 0)
{
System.Console.Out.WriteLine("Queue acls for user : "******"\nQueue Operations");
System.Console.Out.WriteLine("=====================");
foreach (QueueAclsInfo queueInfo in queueAclsInfoList)
{
System.Console.Out.Write(queueInfo.GetQueueName() + " ");
string[] ops = queueInfo.GetOperations();
Arrays.Sort(ops);
int max = ops.Length - 1;
for (int j = 0; j < ops.Length; j++)
{
System.Console.Out.Write(ops[j].ReplaceFirst("acl-", string.Empty));
if (j < max)
{
System.Console.Out.Write(",");
}
}
System.Console.Out.WriteLine();
}
}
else
{
System.Console.Out.WriteLine("User " + ugi.GetShortUserName() + " does not have access to any queue. \n"
);
}
}
/// <exception cref="System.IO.IOException"/>
private UserGroupInformation CheckAcls(string method)
{
UserGroupInformation user;
try
{
user = UserGroupInformation.GetCurrentUser();
}
catch (IOException ioe)
{
Log.Warn("Couldn't get current user", ioe);
HSAuditLogger.LogFailure("UNKNOWN", method, adminAcl.ToString(), HistoryAdminServer
, "Couldn't get current user");
throw;
}
if (!adminAcl.IsUserAllowed(user))
{
Log.Warn("User " + user.GetShortUserName() + " doesn't have permission" + " to call '"
+ method + "'");
HSAuditLogger.LogFailure(user.GetShortUserName(), method, adminAcl.ToString(), HistoryAdminServer
, HSAuditLogger.AuditConstants.UnauthorizedUser);
throw new AccessControlException("User " + user.GetShortUserName() + " doesn't have permission"
+ " to call '" + method + "'");
}
Log.Info("HS Admin: " + method + " invoked by user " + user.GetShortUserName());
return(user);
}
public virtual void TestUsernameExtractionCurrentuser()
{
string whoami = RegistryUtils.GetCurrentUsernameUnencoded(string.Empty);
string ugiUser = UserGroupInformation.GetCurrentUser().GetShortUserName();
NUnit.Framework.Assert.AreEqual(ugiUser, whoami);
}
public static void SetupCluster()
{
Configuration conf = new Configuration();
conf.SetBoolean(DFSConfigKeys.DfsWebhdfsEnabledKey, true);
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024);
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
cluster.WaitActive();
//change root permission to 777
cluster.GetFileSystem().SetPermission(new Path("/"), new FsPermission((short)0x1ff
));
string uri = WebHdfsFileSystem.Scheme + "://" + conf.Get(DFSConfigKeys.DfsNamenodeHttpAddressKey
);
//get file system as a non-superuser
UserGroupInformation current = UserGroupInformation.GetCurrentUser();
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(current.GetShortUserName
() + "x", new string[] { "user" });
fileSystem = ugi.DoAs(new _PrivilegedExceptionAction_91(uri, conf));
defaultWorkingDirectory = fileSystem.GetWorkingDirectory();
}
catch (Exception e)
{
throw new RuntimeException(e);
}
}
/// <summary>
/// Utility method to verify if the current user has access based on the
/// passed
/// <see cref="Org.Apache.Hadoop.Security.Authorize.AccessControlList"/>
/// </summary>
/// <param name="authorizer">
/// the
/// <see cref="Org.Apache.Hadoop.Security.Authorize.AccessControlList"/>
/// to check against
/// </param>
/// <param name="method">the method name to be logged</param>
/// <param name="module">like AdminService or NodeLabelManager</param>
/// <param name="Log">the logger to use</param>
/// <returns>
///
/// <see cref="Org.Apache.Hadoop.Security.UserGroupInformation"/>
/// of the current user
/// </returns>
/// <exception cref="System.IO.IOException"/>
public static UserGroupInformation VerifyAdminAccess(YarnAuthorizationProvider authorizer
, string method, string module, Log Log)
{
UserGroupInformation user;
try
{
user = UserGroupInformation.GetCurrentUser();
}
catch (IOException ioe)
{
Log.Warn("Couldn't get current user", ioe);
RMAuditLogger.LogFailure("UNKNOWN", method, string.Empty, "AdminService", "Couldn't get current user"
);
throw;
}
if (!authorizer.IsAdmin(user))
{
Log.Warn("User " + user.GetShortUserName() + " doesn't have permission" + " to call '"
+ method + "'");
RMAuditLogger.LogFailure(user.GetShortUserName(), method, string.Empty, module, RMAuditLogger.AuditConstants
.UnauthorizedUser);
throw new AccessControlException("User " + user.GetShortUserName() + " doesn't have permission"
+ " to call '" + method + "'");
}
if (Log.IsTraceEnabled())
{
Log.Trace(method + " invoked by user " + user.GetShortUserName());
}
return(user);
}
public virtual void TestUse_ExistingEntry_DupId()
{
// Pre-populate the SCM with one cache entry
store.AddResource("key1", "foo.jar");
UserGroupInformation testUGI = UserGroupInformation.GetCurrentUser();
store.AddResourceReference("key1", new SharedCacheResourceReference(CreateAppId(1
, 1L), testUGI.GetShortUserName()));
NUnit.Framework.Assert.AreEqual(1, store.GetResourceReferences("key1").Count);
long hits = ClientSCMMetrics.GetInstance().GetCacheHits();
// Add a new duplicate appId
UseSharedCacheResourceRequest request = recordFactory.NewRecordInstance <UseSharedCacheResourceRequest
>();
request.SetResourceKey("key1");
request.SetAppId(CreateAppId(1, 1L));
// Expecting default depth of 3 under the shared cache root dir
string expectedPath = testDir.GetAbsolutePath() + "/k/e/y/key1/foo.jar";
NUnit.Framework.Assert.AreEqual(expectedPath, clientSCMProxy.Use(request).GetPath
());
NUnit.Framework.Assert.AreEqual(1, store.GetResourceReferences("key1").Count);
NUnit.Framework.Assert.AreEqual("Client SCM metrics aren't updated.", 1, ClientSCMMetrics
.GetInstance().GetCacheHits() - hits);
}
//@Test (timeout = 60000)
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
public virtual void TestSleepJobWithSecurityOn()
{
Log.Info("\n\n\nStarting testSleepJobWithSecurityOn().");
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
return;
}
mrCluster.GetConfig().Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication
, "kerberos");
mrCluster.GetConfig().Set(YarnConfiguration.RmKeytab, "/etc/krb5.keytab");
mrCluster.GetConfig().Set(YarnConfiguration.NmKeytab, "/etc/krb5.keytab");
mrCluster.GetConfig().Set(YarnConfiguration.RmPrincipal, "rm/sightbusy-lx@LOCALHOST"
);
mrCluster.GetConfig().Set(YarnConfiguration.NmPrincipal, "nm/sightbusy-lx@LOCALHOST"
);
UserGroupInformation.SetConfiguration(mrCluster.GetConfig());
// Keep it in here instead of after RM/NM as multiple user logins happen in
// the same JVM.
UserGroupInformation user = UserGroupInformation.GetCurrentUser();
Log.Info("User name is " + user.GetUserName());
foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> str in user.GetTokens
())
{
Log.Info("Token is " + str.EncodeToUrlString());
}
user.DoAs(new _PrivilegedExceptionAction_552());
}
/// <exception cref="System.Exception"/>
public Void Run()
{
// make sure it is not the same as the login user because we use the
// same UGI object for every instantiation of the login user and you
// won't run into the race condition otherwise
Assert.AssertNotEquals(UserGroupInformation.GetLoginUser(), UserGroupInformation.
GetCurrentUser());
TestUserGroupInformation.GetTokenThread thread = new TestUserGroupInformation.GetTokenThread
();
try
{
thread.Start();
for (int i = 0; i < 100; i++)
{
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> t = Org.Mockito.Mockito.Mock
<Org.Apache.Hadoop.Security.Token.Token>();
Org.Mockito.Mockito.When(t.GetService()).ThenReturn(new Text("t" + i));
UserGroupInformation.GetCurrentUser().AddToken(t);
NUnit.Framework.Assert.IsNull("ConcurrentModificationException encountered", thread
.cme);
}
}
catch (ConcurrentModificationException cme)
{
Runtime.PrintStackTrace(cme);
NUnit.Framework.Assert.Fail("ConcurrentModificationException encountered");
}
finally
{
thread.runThread = false;
thread.Join(5 * 1000);
}
return(null);
}
/// <exception cref="System.IO.IOException"/>
private int GetGroups(string[] usernames)
{
// Get groups users belongs to
if (usernames.Length == 0)
{
usernames = new string[] { UserGroupInformation.GetCurrentUser().GetUserName() };
}
// Get the current configuration
Configuration conf = GetConf();
IPEndPoint address = conf.GetSocketAddr(JHAdminConfig.JhsAdminAddress, JHAdminConfig
.DefaultJhsAdminAddress, JHAdminConfig.DefaultJhsAdminPort);
GetUserMappingsProtocol getUserMappingProtocol = HSProxies.CreateProxy <GetUserMappingsProtocol
>(conf, address, UserGroupInformation.GetCurrentUser());
foreach (string username in usernames)
{
StringBuilder sb = new StringBuilder();
sb.Append(username + " :");
foreach (string group in getUserMappingProtocol.GetGroupsForUser(username))
{
sb.Append(" ");
sb.Append(group);
}
System.Console.Out.WriteLine(sb);
}
return(0);
}
/// <exception cref="System.IO.IOException"/>
public Void Run()
{
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
NUnit.Framework.Assert.IsNotNull(ugi);
return(null);
}
public virtual void TestDelegationTokenRpc()
{
ClientProtocol mockNN = Org.Mockito.Mockito.Mock <ClientProtocol>();
FSNamesystem mockNameSys = Org.Mockito.Mockito.Mock <FSNamesystem>();
DelegationTokenSecretManager sm = new DelegationTokenSecretManager(DFSConfigKeys.
DfsNamenodeDelegationKeyUpdateIntervalDefault, DFSConfigKeys.DfsNamenodeDelegationKeyUpdateIntervalDefault
, DFSConfigKeys.DfsNamenodeDelegationTokenMaxLifetimeDefault, 3600000, mockNameSys
);
sm.StartThreads();
Org.Apache.Hadoop.Ipc.Server server = new RPC.Builder(conf).SetProtocol(typeof(ClientProtocol
)).SetInstance(mockNN).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose
(true).SetSecretManager(sm).Build();
server.Start();
UserGroupInformation current = UserGroupInformation.GetCurrentUser();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
string user = current.GetUserName();
Text owner = new Text(user);
DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(owner, owner, null
);
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>(dtId, sm);
SecurityUtil.SetTokenService(token, addr);
Log.Info("Service for token is " + token.GetService());
current.AddToken(token);
current.DoAs(new _PrivilegedExceptionAction_100(addr, server));
}
/// <exception cref="System.Exception"/>
private static void UploadEmptyContainerLogIntoRemoteDir(UserGroupInformation ugi
, Configuration configuration, IList <string> rootLogDirs, NodeId nodeId, ContainerId
containerId, Path appDir, FileSystem fs)
{
Path path = new Path(appDir, LogAggregationUtils.GetNodeString(nodeId) + Runtime.
CurrentTimeMillis());
AggregatedLogFormat.LogWriter writer = new AggregatedLogFormat.LogWriter(configuration
, path, ugi);
writer.WriteApplicationOwner(ugi.GetUserName());
IDictionary <ApplicationAccessType, string> appAcls = new Dictionary <ApplicationAccessType
, string>();
appAcls[ApplicationAccessType.ViewApp] = ugi.GetUserName();
writer.WriteApplicationACLs(appAcls);
DataOutputStream @out = writer.GetWriter().PrepareAppendKey(-1);
new AggregatedLogFormat.LogKey(containerId).Write(@out);
@out.Close();
@out = writer.GetWriter().PrepareAppendValue(-1);
new AggregatedLogFormat.LogValue(rootLogDirs, containerId, UserGroupInformation.GetCurrentUser
().GetShortUserName()).Write(@out, new HashSet <FilePath>());
@out.Close();
writer.Close();
}
// FIXME:
// Disabled this test because currently, when shutdown hook triggered at
// lastRetry in RM view, cleanup will not do. This should be supported after
// YARN-2261 completed
// @Test (timeout = 30000)
/// <exception cref="System.IO.IOException"/>
public virtual void TestDeletionofStagingOnKillLastTry()
{
conf.Set(MRJobConfig.MapreduceJobDir, stagingJobDir);
fs = Org.Mockito.Mockito.Mock <FileSystem>();
Org.Mockito.Mockito.When(fs.Delete(Matchers.Any <Path>(), Matchers.AnyBoolean())).
ThenReturn(true);
//Staging Dir exists
string user = UserGroupInformation.GetCurrentUser().GetShortUserName();
Path stagingDir = MRApps.GetStagingAreaDir(conf, user);
Org.Mockito.Mockito.When(fs.Exists(stagingDir)).ThenReturn(true);
ApplicationId appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 0);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
JobId jobid = recordFactory.NewRecordInstance <JobId>();
jobid.SetAppId(appId);
ContainerAllocator mockAlloc = Org.Mockito.Mockito.Mock <ContainerAllocator>();
MRAppMaster appMaster = new TestStagingCleanup.TestMRApp(this, attemptId, mockAlloc
);
//no retry
appMaster.Init(conf);
NUnit.Framework.Assert.IsTrue("appMaster.isLastAMRetry() is false", appMaster.IsLastAMRetry
());
//simulate the process being killed
MRAppMaster.MRAppMasterShutdownHook hook = new MRAppMaster.MRAppMasterShutdownHook
(appMaster);
hook.Run();
NUnit.Framework.Assert.IsTrue("MRAppMaster isn't stopped", appMaster.IsInState(Service.STATE
.Stopped));
Org.Mockito.Mockito.Verify(fs).Delete(stagingJobPath, true);
}
Answer(InvocationOnMock invocation)
{
object[] args = invocation.GetArguments();
NUnit.Framework.Assert.AreEqual(2, args.Length);
ClientDatanodeProtocolProtos.GetReplicaVisibleLengthRequestProto req = (ClientDatanodeProtocolProtos.GetReplicaVisibleLengthRequestProto
)args[1];
ICollection <TokenIdentifier> tokenIds = UserGroupInformation.GetCurrentUser().GetTokenIdentifiers
();
NUnit.Framework.Assert.AreEqual("Only one BlockTokenIdentifier expected", 1, tokenIds
.Count);
long result = 0;
foreach (TokenIdentifier tokenId in tokenIds)
{
BlockTokenIdentifier id = (BlockTokenIdentifier)tokenId;
Log.Info("Got: " + id.ToString());
NUnit.Framework.Assert.IsTrue("Received BlockTokenIdentifier is wrong", ident.Equals
(id));
sm.CheckAccess(id, null, PBHelper.Convert(req.GetBlock()), BlockTokenSecretManager.AccessMode
.Write);
result = id.GetBlockId();
}
return((ClientDatanodeProtocolProtos.GetReplicaVisibleLengthResponseProto)ClientDatanodeProtocolProtos.GetReplicaVisibleLengthResponseProto
.NewBuilder().SetLength(result).Build());
}
public virtual void TestDeletionofStaging()
{
conf.Set(MRJobConfig.MapreduceJobDir, stagingJobDir);
fs = Org.Mockito.Mockito.Mock <FileSystem>();
Org.Mockito.Mockito.When(fs.Delete(Matchers.Any <Path>(), Matchers.AnyBoolean())).
ThenReturn(true);
//Staging Dir exists
string user = UserGroupInformation.GetCurrentUser().GetShortUserName();
Path stagingDir = MRApps.GetStagingAreaDir(conf, user);
Org.Mockito.Mockito.When(fs.Exists(stagingDir)).ThenReturn(true);
ApplicationId appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 0);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
JobId jobid = recordFactory.NewRecordInstance <JobId>();
jobid.SetAppId(appId);
ContainerAllocator mockAlloc = Org.Mockito.Mockito.Mock <ContainerAllocator>();
NUnit.Framework.Assert.IsTrue(MRJobConfig.DefaultMrAmMaxAttempts > 1);
MRAppMaster appMaster = new TestStagingCleanup.TestMRApp(this, attemptId, mockAlloc
, JobStateInternal.Running, MRJobConfig.DefaultMrAmMaxAttempts);
appMaster.Init(conf);
appMaster.Start();
appMaster.ShutDownJob();
//test whether notifyIsLastAMRetry called
NUnit.Framework.Assert.AreEqual(true, ((TestStagingCleanup.TestMRApp)appMaster).GetTestIsLastAMRetry
());
Org.Mockito.Mockito.Verify(fs).Delete(stagingJobPath, true);
}
protected override void ConfigureServlets()
{
this.Bind <JAXBContextResolver>();
this.Bind <RMWebServices>();
this.Bind <GenericExceptionHandler>();
try
{
this._enclosing.userName = UserGroupInformation.GetCurrentUser().GetShortUserName
();
}
catch (IOException ioe)
{
throw new RuntimeException("Unable to get current user name " + ioe.Message, ioe);
}
this._enclosing.notUserName = this._enclosing.userName + "abc123";
this._enclosing.conf = new YarnConfiguration();
this._enclosing.conf.Set(YarnConfiguration.YarnAdminAcl, this._enclosing.userName
);
Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Webapp.TestRMWebServicesNodeLabels.
rm = new MockRM(this._enclosing.conf);
this.Bind <ResourceManager>().ToInstance(Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Webapp.TestRMWebServicesNodeLabels
.rm);
this.Filter("/*").Through(typeof(TestRMWebServicesAppsModification.TestRMCustomAuthFilter
));
this.Serve("/*").With(typeof(GuiceContainer));
}
public virtual void TestRefreshAdminAcls()
{
// Setting current user to admin acl
conf.Set(JHAdminConfig.JhsAdminAcl, UserGroupInformation.GetCurrentUser().GetUserName
());
string[] args = new string[1];
args[0] = "-refreshAdminAcls";
hsAdminClient.Run(args);
// Now I should be able to run any hsadmin command without any exception
// being thrown
args[0] = "-refreshSuperUserGroupsConfiguration";
hsAdminClient.Run(args);
// Lets remove current user from admin acl
conf.Set(JHAdminConfig.JhsAdminAcl, "notCurrentUser");
args[0] = "-refreshAdminAcls";
hsAdminClient.Run(args);
// Now I should get an exception if i run any hsadmin command
Exception th = null;
args[0] = "-refreshSuperUserGroupsConfiguration";
try
{
hsAdminClient.Run(args);
}
catch (Exception e)
{
th = e;
}
NUnit.Framework.Assert.IsTrue(th is RemoteException);
}