/// <exception cref="System.Exception"/> public virtual void TestAddCreds <T>() where T : TokenIdentifier { // from Mockito mocks UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("someone"); Text service = new Text("service"); Org.Apache.Hadoop.Security.Token.Token <T> t1 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token >(); Org.Mockito.Mockito.When(t1.GetService()).ThenReturn(service); Org.Apache.Hadoop.Security.Token.Token <T> t2 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token >(); Org.Mockito.Mockito.When(t2.GetService()).ThenReturn(new Text("service2")); byte[] secret = new byte[] { }; Text secretKey = new Text("sshhh"); // fill credentials Credentials creds = new Credentials(); creds.AddToken(t1.GetService(), t1); creds.AddToken(t2.GetService(), t2); creds.AddSecretKey(secretKey, secret); // add creds to ugi, and check ugi ugi.AddCredentials(creds); CheckTokens(ugi, t1, t2); NUnit.Framework.Assert.AreSame(secret, ugi.GetCredentials().GetSecretKey(secretKey )); }
public override void Flush() { lock (this) { user.AddCredentials(credentials); } }
protected internal virtual void InitAppAggregator(ApplicationId appId, string user , Credentials credentials, ContainerLogsRetentionPolicy logRetentionPolicy, IDictionary <ApplicationAccessType, string> appAcls, LogAggregationContext logAggregationContext ) { // Get user's FileSystem credentials UserGroupInformation userUgi = UserGroupInformation.CreateRemoteUser(user); if (credentials != null) { userUgi.AddCredentials(credentials); } // New application AppLogAggregator appLogAggregator = new AppLogAggregatorImpl(this.dispatcher, this .deletionService, GetConfig(), appId, userUgi, this.nodeId, dirsHandler, GetRemoteNodeLogFileForApp (appId, user), logRetentionPolicy, appAcls, logAggregationContext, this.context, GetLocalFileContext(GetConfig())); if (this.appLogAggregators.PutIfAbsent(appId, appLogAggregator) != null) { throw new YarnRuntimeException("Duplicate initApp for " + appId); } // wait until check for existing aggregator to create dirs YarnRuntimeException appDirException = null; try { // Create the app dir CreateAppDir(user, appId, userUgi); } catch (Exception e) { appLogAggregator.DisableLogAggregation(); if (!(e is YarnRuntimeException)) { appDirException = new YarnRuntimeException(e); } else { appDirException = (YarnRuntimeException)e; } } // TODO Get the user configuration for the list of containers that need log // aggregation. // Schedule the aggregator. Runnable aggregatorWrapper = new _Runnable_381(this, appLogAggregator, appId, userUgi ); this.threadPool.Execute(aggregatorWrapper); if (appDirException != null) { throw appDirException; } }
/// <exception cref="System.Exception"/> public virtual void TestUGITokens <T>() where T : TokenIdentifier { // from Mockito mocks UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting("TheDoctor", new string[] { "TheTARDIS" }); Org.Apache.Hadoop.Security.Token.Token <T> t1 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token >(); Org.Mockito.Mockito.When(t1.GetService()).ThenReturn(new Text("t1")); Org.Apache.Hadoop.Security.Token.Token <T> t2 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Security.Token.Token >(); Org.Mockito.Mockito.When(t2.GetService()).ThenReturn(new Text("t2")); Credentials creds = new Credentials(); byte[] secretKey = new byte[] { }; Text secretName = new Text("shhh"); creds.AddSecretKey(secretName, secretKey); ugi.AddToken(t1); ugi.AddToken(t2); ugi.AddCredentials(creds); ICollection <Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> > z = ugi.GetTokens (); Assert.True(z.Contains(t1)); Assert.True(z.Contains(t2)); Assert.Equal(2, z.Count); Credentials ugiCreds = ugi.GetCredentials(); NUnit.Framework.Assert.AreSame(secretKey, ugiCreds.GetSecretKey(secretName)); Assert.Equal(1, ugiCreds.NumberOfSecretKeys()); try { z.Remove(t1); NUnit.Framework.Assert.Fail("Shouldn't be able to modify token collection from UGI" ); } catch (NotSupportedException) { } // Can't modify tokens // ensure that the tokens are passed through doAs ICollection <Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> > otherSet = ugi .DoAs(new _PrivilegedExceptionAction_612()); Assert.True(otherSet.Contains(t1)); Assert.True(otherSet.Contains(t2)); }
public virtual void TestAddTokensToUGI() { UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("someone"); Credentials creds = new Credentials(); for (int i = 0; i < service.Length; i++) { creds.AddToken(service[i], token[i]); } ugi.AddCredentials(creds); creds = ugi.GetCredentials(); for (int i_1 = 0; i_1 < service.Length; i_1++) { NUnit.Framework.Assert.AreSame(token[i_1], creds.GetToken(service[i_1])); } Assert.Equal(service.Length, creds.NumberOfTokens()); }
/// <exception cref="System.Exception"/> public static void Main(string[] args) { Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler ()); Log.Debug("Child starting"); JobConf job = new JobConf(MRJobConfig.JobConfFile); // Initing with our JobConf allows us to avoid loading confs twice Limits.Init(job); UserGroupInformation.SetConfiguration(job); string host = args[0]; int port = System.Convert.ToInt32(args[1]); IPEndPoint address = NetUtils.CreateSocketAddrForHost(host, port); TaskAttemptID firstTaskid = ((TaskAttemptID)TaskAttemptID.ForName(args[2])); long jvmIdLong = long.Parse(args[3]); JVMId jvmId = new JVMId(((JobID)firstTaskid.GetJobID()), firstTaskid.GetTaskType( ) == TaskType.Map, jvmIdLong); // initialize metrics DefaultMetricsSystem.Initialize(StringUtils.Camelize(firstTaskid.GetTaskType().ToString ()) + "Task"); // Security framework already loaded the tokens into current ugi Credentials credentials = UserGroupInformation.GetCurrentUser().GetCredentials(); Log.Info("Executing with tokens:"); foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in credentials.GetAllTokens ()) { Log.Info(token); } // Create TaskUmbilicalProtocol as actual task owner. UserGroupInformation taskOwner = UserGroupInformation.CreateRemoteUser(((JobID)firstTaskid .GetJobID()).ToString()); Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jt = TokenCache.GetJobToken (credentials); SecurityUtil.SetTokenService(jt, address); taskOwner.AddToken(jt); TaskUmbilicalProtocol umbilical = taskOwner.DoAs(new _PrivilegedExceptionAction_108 (address, job)); // report non-pid to application master JvmContext context = new JvmContext(jvmId, "-1000"); Log.Debug("PID: " + Sharpen.Runtime.GetEnv()["JVM_PID"]); Task task = null; UserGroupInformation childUGI = null; ScheduledExecutorService logSyncer = null; try { int idleLoopCount = 0; JvmTask myTask = null; // poll for new task for (int idle = 0; null == myTask; ++idle) { long sleepTimeMilliSecs = Math.Min(idle * 500, 1500); Log.Info("Sleeping for " + sleepTimeMilliSecs + "ms before retrying again. Got null now." ); TimeUnit.Milliseconds.Sleep(sleepTimeMilliSecs); myTask = umbilical.GetTask(context); } if (myTask.ShouldDie()) { return; } task = myTask.GetTask(); YarnChild.taskid = task.GetTaskID(); // Create the job-conf and set credentials ConfigureTask(job, task, credentials, jt); // Initiate Java VM metrics JvmMetrics.InitSingleton(jvmId.ToString(), job.GetSessionId()); childUGI = UserGroupInformation.CreateRemoteUser(Runtime.Getenv(ApplicationConstants.Environment .User.ToString())); // Add tokens to new user so that it may execute its task correctly. childUGI.AddCredentials(credentials); // set job classloader if configured before invoking the task MRApps.SetJobClassLoader(job); logSyncer = TaskLog.CreateLogSyncer(); // Create a final reference to the task for the doAs block Task taskFinal = task; childUGI.DoAs(new _PrivilegedExceptionAction_158(taskFinal, job, umbilical)); } catch (FSError e) { // use job-specified working directory // run the task Log.Fatal("FSError from child", e); if (!ShutdownHookManager.Get().IsShutdownInProgress()) { umbilical.FsError(taskid, e.Message); } } catch (Exception exception) { Log.Warn("Exception running child : " + StringUtils.StringifyException(exception) ); try { if (task != null) { // do cleanup for the task if (childUGI == null) { // no need to job into doAs block task.TaskCleanup(umbilical); } else { Task taskFinal = task; childUGI.DoAs(new _PrivilegedExceptionAction_183(taskFinal, umbilical)); } } } catch (Exception e) { Log.Info("Exception cleaning up: " + StringUtils.StringifyException(e)); } // Report back any failures, for diagnostic purposes if (taskid != null) { if (!ShutdownHookManager.Get().IsShutdownInProgress()) { umbilical.FatalError(taskid, StringUtils.StringifyException(exception)); } } } catch (Exception throwable) { Log.Fatal("Error running child : " + StringUtils.StringifyException(throwable)); if (taskid != null) { if (!ShutdownHookManager.Get().IsShutdownInProgress()) { Exception tCause = throwable.InnerException; string cause = tCause == null ? throwable.Message : StringUtils.StringifyException (tCause); umbilical.FatalError(taskid, cause); } } } finally { RPC.StopProxy(umbilical); DefaultMetricsSystem.Shutdown(); TaskLog.SyncLogsShutdown(logSyncer); } }
private void UploadLogsForContainers(bool appFinished) { if (this.logAggregationDisabled) { return; } if (UserGroupInformation.IsSecurityEnabled()) { Credentials systemCredentials = context.GetSystemCredentialsForApps()[appId]; if (systemCredentials != null) { if (Log.IsDebugEnabled()) { Log.Debug("Adding new framework-token for " + appId + " for log-aggregation: " + systemCredentials.GetAllTokens() + "; userUgi=" + userUgi); } // this will replace old token userUgi.AddCredentials(systemCredentials); } } // Create a set of Containers whose logs will be uploaded in this cycle. // It includes: // a) all containers in pendingContainers: those containers are finished // and satisfy the retentionPolicy. // b) some set of running containers: For all the Running containers, // we have ContainerLogsRetentionPolicy.AM_AND_FAILED_CONTAINERS_ONLY, // so simply set wasContainerSuccessful as true to // bypass FAILED_CONTAINERS check and find the running containers // which satisfy the retentionPolicy. ICollection <ContainerId> pendingContainerInThisCycle = new HashSet <ContainerId>(); this.pendingContainers.DrainTo(pendingContainerInThisCycle); ICollection <ContainerId> finishedContainers = new HashSet <ContainerId>(pendingContainerInThisCycle ); if (this.context.GetApplications()[this.appId] != null) { foreach (ContainerId container in this.context.GetApplications()[this.appId].GetContainers ().Keys) { if (ShouldUploadLogs(container, true)) { pendingContainerInThisCycle.AddItem(container); } } } AggregatedLogFormat.LogWriter writer = null; try { try { writer = new AggregatedLogFormat.LogWriter(this.conf, this.remoteNodeTmpLogFileForApp , this.userUgi); // Write ACLs once when the writer is created. writer.WriteApplicationACLs(appAcls); writer.WriteApplicationOwner(this.userUgi.GetShortUserName()); } catch (IOException e1) { Log.Error("Cannot create writer for app " + this.applicationId + ". Skip log upload this time. " , e1); return; } bool uploadedLogsInThisCycle = false; foreach (ContainerId container in pendingContainerInThisCycle) { AppLogAggregatorImpl.ContainerLogAggregator aggregator = null; if (containerLogAggregators.Contains(container)) { aggregator = containerLogAggregators[container]; } else { aggregator = new AppLogAggregatorImpl.ContainerLogAggregator(this, container); containerLogAggregators[container] = aggregator; } ICollection <Path> uploadedFilePathsInThisCycle = aggregator.DoContainerLogAggregation (writer, appFinished); if (uploadedFilePathsInThisCycle.Count > 0) { uploadedLogsInThisCycle = true; this.delService.Delete(this.userUgi.GetShortUserName(), null, Sharpen.Collections.ToArray (uploadedFilePathsInThisCycle, new Path[uploadedFilePathsInThisCycle.Count])); } // This container is finished, and all its logs have been uploaded, // remove it from containerLogAggregators. if (finishedContainers.Contains(container)) { Sharpen.Collections.Remove(containerLogAggregators, container); } } // Before upload logs, make sure the number of existing logs // is smaller than the configured NM log aggregation retention size. if (uploadedLogsInThisCycle) { CleanOldLogs(); } if (writer != null) { writer.Close(); writer = null; } Path renamedPath = this.rollingMonitorInterval <= 0 ? remoteNodeLogFileForApp : new Path(remoteNodeLogFileForApp.GetParent(), remoteNodeLogFileForApp.GetName() + "_" + Runtime.CurrentTimeMillis()); bool rename = uploadedLogsInThisCycle; try { userUgi.DoAs(new _PrivilegedExceptionAction_304(this, rename, renamedPath)); } catch (Exception e) { Log.Error("Failed to move temporary log file to final location: [" + remoteNodeTmpLogFileForApp + "] to [" + renamedPath + "]", e); } } finally { if (writer != null) { writer.Close(); } } }