/// <exception cref="System.Exception"/>
public virtual void TestAddCreds <T>()
where T : TokenIdentifier
{
// from Mockito mocks
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("someone");
Text service = new Text("service");
Org.Apache.Hadoop.Security.Token.Token <T> t1 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Org.Mockito.Mockito.When(t1.GetService()).ThenReturn(service);
Org.Apache.Hadoop.Security.Token.Token <T> t2 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Org.Mockito.Mockito.When(t2.GetService()).ThenReturn(new Text("service2"));
byte[] secret = new byte[] { };
Text secretKey = new Text("sshhh");
// fill credentials
Credentials creds = new Credentials();
creds.AddToken(t1.GetService(), t1);
creds.AddToken(t2.GetService(), t2);
creds.AddSecretKey(secretKey, secret);
// add creds to ugi, and check ugi
ugi.AddCredentials(creds);
CheckTokens(ugi, t1, t2);
NUnit.Framework.Assert.AreSame(secret, ugi.GetCredentials().GetSecretKey(secretKey
));
}
public override void Flush()
{
lock (this)
{
user.AddCredentials(credentials);
}
}
protected internal virtual void InitAppAggregator(ApplicationId appId, string user
, Credentials credentials, ContainerLogsRetentionPolicy logRetentionPolicy, IDictionary
<ApplicationAccessType, string> appAcls, LogAggregationContext logAggregationContext
)
{
// Get user's FileSystem credentials
UserGroupInformation userUgi = UserGroupInformation.CreateRemoteUser(user);
if (credentials != null)
{
userUgi.AddCredentials(credentials);
}
// New application
AppLogAggregator appLogAggregator = new AppLogAggregatorImpl(this.dispatcher, this
.deletionService, GetConfig(), appId, userUgi, this.nodeId, dirsHandler, GetRemoteNodeLogFileForApp
(appId, user), logRetentionPolicy, appAcls, logAggregationContext, this.context,
GetLocalFileContext(GetConfig()));
if (this.appLogAggregators.PutIfAbsent(appId, appLogAggregator) != null)
{
throw new YarnRuntimeException("Duplicate initApp for " + appId);
}
// wait until check for existing aggregator to create dirs
YarnRuntimeException appDirException = null;
try
{
// Create the app dir
CreateAppDir(user, appId, userUgi);
}
catch (Exception e)
{
appLogAggregator.DisableLogAggregation();
if (!(e is YarnRuntimeException))
{
appDirException = new YarnRuntimeException(e);
}
else
{
appDirException = (YarnRuntimeException)e;
}
}
// TODO Get the user configuration for the list of containers that need log
// aggregation.
// Schedule the aggregator.
Runnable aggregatorWrapper = new _Runnable_381(this, appLogAggregator, appId, userUgi
);
this.threadPool.Execute(aggregatorWrapper);
if (appDirException != null)
{
throw appDirException;
}
}
/// <exception cref="System.Exception"/>
public virtual void TestUGITokens <T>()
where T : TokenIdentifier
{
// from Mockito mocks
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting("TheDoctor",
new string[] { "TheTARDIS" });
Org.Apache.Hadoop.Security.Token.Token <T> t1 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Org.Mockito.Mockito.When(t1.GetService()).ThenReturn(new Text("t1"));
Org.Apache.Hadoop.Security.Token.Token <T> t2 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token
>();
Org.Mockito.Mockito.When(t2.GetService()).ThenReturn(new Text("t2"));
Credentials creds = new Credentials();
byte[] secretKey = new byte[] { };
Text secretName = new Text("shhh");
creds.AddSecretKey(secretName, secretKey);
ugi.AddToken(t1);
ugi.AddToken(t2);
ugi.AddCredentials(creds);
ICollection <Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> > z = ugi.GetTokens
();
Assert.True(z.Contains(t1));
Assert.True(z.Contains(t2));
Assert.Equal(2, z.Count);
Credentials ugiCreds = ugi.GetCredentials();
NUnit.Framework.Assert.AreSame(secretKey, ugiCreds.GetSecretKey(secretName));
Assert.Equal(1, ugiCreds.NumberOfSecretKeys());
try
{
z.Remove(t1);
NUnit.Framework.Assert.Fail("Shouldn't be able to modify token collection from UGI"
);
}
catch (NotSupportedException)
{
}
// Can't modify tokens
// ensure that the tokens are passed through doAs
ICollection <Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> > otherSet = ugi
.DoAs(new _PrivilegedExceptionAction_612());
Assert.True(otherSet.Contains(t1));
Assert.True(otherSet.Contains(t2));
}
public virtual void TestAddTokensToUGI()
{
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("someone");
Credentials creds = new Credentials();
for (int i = 0; i < service.Length; i++)
{
creds.AddToken(service[i], token[i]);
}
ugi.AddCredentials(creds);
creds = ugi.GetCredentials();
for (int i_1 = 0; i_1 < service.Length; i_1++)
{
NUnit.Framework.Assert.AreSame(token[i_1], creds.GetToken(service[i_1]));
}
Assert.Equal(service.Length, creds.NumberOfTokens());
}
/// <exception cref="System.Exception"/>
public static void Main(string[] args)
{
Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler
());
Log.Debug("Child starting");
JobConf job = new JobConf(MRJobConfig.JobConfFile);
// Initing with our JobConf allows us to avoid loading confs twice
Limits.Init(job);
UserGroupInformation.SetConfiguration(job);
string host = args[0];
int port = System.Convert.ToInt32(args[1]);
IPEndPoint address = NetUtils.CreateSocketAddrForHost(host, port);
TaskAttemptID firstTaskid = ((TaskAttemptID)TaskAttemptID.ForName(args[2]));
long jvmIdLong = long.Parse(args[3]);
JVMId jvmId = new JVMId(((JobID)firstTaskid.GetJobID()), firstTaskid.GetTaskType(
) == TaskType.Map, jvmIdLong);
// initialize metrics
DefaultMetricsSystem.Initialize(StringUtils.Camelize(firstTaskid.GetTaskType().ToString
()) + "Task");
// Security framework already loaded the tokens into current ugi
Credentials credentials = UserGroupInformation.GetCurrentUser().GetCredentials();
Log.Info("Executing with tokens:");
foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in credentials.GetAllTokens
())
{
Log.Info(token);
}
// Create TaskUmbilicalProtocol as actual task owner.
UserGroupInformation taskOwner = UserGroupInformation.CreateRemoteUser(((JobID)firstTaskid
.GetJobID()).ToString());
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jt = TokenCache.GetJobToken
(credentials);
SecurityUtil.SetTokenService(jt, address);
taskOwner.AddToken(jt);
TaskUmbilicalProtocol umbilical = taskOwner.DoAs(new _PrivilegedExceptionAction_108
(address, job));
// report non-pid to application master
JvmContext context = new JvmContext(jvmId, "-1000");
Log.Debug("PID: " + Sharpen.Runtime.GetEnv()["JVM_PID"]);
Task task = null;
UserGroupInformation childUGI = null;
ScheduledExecutorService logSyncer = null;
try
{
int idleLoopCount = 0;
JvmTask myTask = null;
// poll for new task
for (int idle = 0; null == myTask; ++idle)
{
long sleepTimeMilliSecs = Math.Min(idle * 500, 1500);
Log.Info("Sleeping for " + sleepTimeMilliSecs + "ms before retrying again. Got null now."
);
TimeUnit.Milliseconds.Sleep(sleepTimeMilliSecs);
myTask = umbilical.GetTask(context);
}
if (myTask.ShouldDie())
{
return;
}
task = myTask.GetTask();
YarnChild.taskid = task.GetTaskID();
// Create the job-conf and set credentials
ConfigureTask(job, task, credentials, jt);
// Initiate Java VM metrics
JvmMetrics.InitSingleton(jvmId.ToString(), job.GetSessionId());
childUGI = UserGroupInformation.CreateRemoteUser(Runtime.Getenv(ApplicationConstants.Environment
.User.ToString()));
// Add tokens to new user so that it may execute its task correctly.
childUGI.AddCredentials(credentials);
// set job classloader if configured before invoking the task
MRApps.SetJobClassLoader(job);
logSyncer = TaskLog.CreateLogSyncer();
// Create a final reference to the task for the doAs block
Task taskFinal = task;
childUGI.DoAs(new _PrivilegedExceptionAction_158(taskFinal, job, umbilical));
}
catch (FSError e)
{
// use job-specified working directory
// run the task
Log.Fatal("FSError from child", e);
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
umbilical.FsError(taskid, e.Message);
}
}
catch (Exception exception)
{
Log.Warn("Exception running child : " + StringUtils.StringifyException(exception)
);
try
{
if (task != null)
{
// do cleanup for the task
if (childUGI == null)
{
// no need to job into doAs block
task.TaskCleanup(umbilical);
}
else
{
Task taskFinal = task;
childUGI.DoAs(new _PrivilegedExceptionAction_183(taskFinal, umbilical));
}
}
}
catch (Exception e)
{
Log.Info("Exception cleaning up: " + StringUtils.StringifyException(e));
}
// Report back any failures, for diagnostic purposes
if (taskid != null)
{
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
umbilical.FatalError(taskid, StringUtils.StringifyException(exception));
}
}
}
catch (Exception throwable)
{
Log.Fatal("Error running child : " + StringUtils.StringifyException(throwable));
if (taskid != null)
{
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
Exception tCause = throwable.InnerException;
string cause = tCause == null ? throwable.Message : StringUtils.StringifyException
(tCause);
umbilical.FatalError(taskid, cause);
}
}
}
finally
{
RPC.StopProxy(umbilical);
DefaultMetricsSystem.Shutdown();
TaskLog.SyncLogsShutdown(logSyncer);
}
}
private void UploadLogsForContainers(bool appFinished)
{
if (this.logAggregationDisabled)
{
return;
}
if (UserGroupInformation.IsSecurityEnabled())
{
Credentials systemCredentials = context.GetSystemCredentialsForApps()[appId];
if (systemCredentials != null)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Adding new framework-token for " + appId + " for log-aggregation: " +
systemCredentials.GetAllTokens() + "; userUgi=" + userUgi);
}
// this will replace old token
userUgi.AddCredentials(systemCredentials);
}
}
// Create a set of Containers whose logs will be uploaded in this cycle.
// It includes:
// a) all containers in pendingContainers: those containers are finished
// and satisfy the retentionPolicy.
// b) some set of running containers: For all the Running containers,
// we have ContainerLogsRetentionPolicy.AM_AND_FAILED_CONTAINERS_ONLY,
// so simply set wasContainerSuccessful as true to
// bypass FAILED_CONTAINERS check and find the running containers
// which satisfy the retentionPolicy.
ICollection <ContainerId> pendingContainerInThisCycle = new HashSet <ContainerId>();
this.pendingContainers.DrainTo(pendingContainerInThisCycle);
ICollection <ContainerId> finishedContainers = new HashSet <ContainerId>(pendingContainerInThisCycle
);
if (this.context.GetApplications()[this.appId] != null)
{
foreach (ContainerId container in this.context.GetApplications()[this.appId].GetContainers
().Keys)
{
if (ShouldUploadLogs(container, true))
{
pendingContainerInThisCycle.AddItem(container);
}
}
}
AggregatedLogFormat.LogWriter writer = null;
try
{
try
{
writer = new AggregatedLogFormat.LogWriter(this.conf, this.remoteNodeTmpLogFileForApp
, this.userUgi);
// Write ACLs once when the writer is created.
writer.WriteApplicationACLs(appAcls);
writer.WriteApplicationOwner(this.userUgi.GetShortUserName());
}
catch (IOException e1)
{
Log.Error("Cannot create writer for app " + this.applicationId + ". Skip log upload this time. "
, e1);
return;
}
bool uploadedLogsInThisCycle = false;
foreach (ContainerId container in pendingContainerInThisCycle)
{
AppLogAggregatorImpl.ContainerLogAggregator aggregator = null;
if (containerLogAggregators.Contains(container))
{
aggregator = containerLogAggregators[container];
}
else
{
aggregator = new AppLogAggregatorImpl.ContainerLogAggregator(this, container);
containerLogAggregators[container] = aggregator;
}
ICollection <Path> uploadedFilePathsInThisCycle = aggregator.DoContainerLogAggregation
(writer, appFinished);
if (uploadedFilePathsInThisCycle.Count > 0)
{
uploadedLogsInThisCycle = true;
this.delService.Delete(this.userUgi.GetShortUserName(), null, Sharpen.Collections.ToArray
(uploadedFilePathsInThisCycle, new Path[uploadedFilePathsInThisCycle.Count]));
}
// This container is finished, and all its logs have been uploaded,
// remove it from containerLogAggregators.
if (finishedContainers.Contains(container))
{
Sharpen.Collections.Remove(containerLogAggregators, container);
}
}
// Before upload logs, make sure the number of existing logs
// is smaller than the configured NM log aggregation retention size.
if (uploadedLogsInThisCycle)
{
CleanOldLogs();
}
if (writer != null)
{
writer.Close();
writer = null;
}
Path renamedPath = this.rollingMonitorInterval <= 0 ? remoteNodeLogFileForApp : new
Path(remoteNodeLogFileForApp.GetParent(), remoteNodeLogFileForApp.GetName() + "_"
+ Runtime.CurrentTimeMillis());
bool rename = uploadedLogsInThisCycle;
try
{
userUgi.DoAs(new _PrivilegedExceptionAction_304(this, rename, renamedPath));
}
catch (Exception e)
{
Log.Error("Failed to move temporary log file to final location: [" + remoteNodeTmpLogFileForApp
+ "] to [" + renamedPath + "]", e);
}
}
finally
{
if (writer != null)
{
writer.Close();
}
}
}