public ContainerTokenIdentifier(ContainerId containerID, string hostName, string
                                 appSubmitter, Resource r, long expiryTimeStamp, int masterKeyId, long rmIdentifier
                                 , Priority priority, long creationTime, LogAggregationContext logAggregationContext
                                 )
 {
     YarnSecurityTokenProtos.ContainerTokenIdentifierProto.Builder builder = YarnSecurityTokenProtos.ContainerTokenIdentifierProto
                                                                             .NewBuilder();
     if (containerID != null)
     {
         builder.SetContainerId(((ContainerIdPBImpl)containerID).GetProto());
     }
     builder.SetNmHostAddr(hostName);
     builder.SetAppSubmitter(appSubmitter);
     if (r != null)
     {
         builder.SetResource(((ResourcePBImpl)r).GetProto());
     }
     builder.SetExpiryTimeStamp(expiryTimeStamp);
     builder.SetMasterKeyId(masterKeyId);
     builder.SetRmIdentifier(rmIdentifier);
     if (priority != null)
     {
         builder.SetPriority(((PriorityPBImpl)priority).GetProto());
     }
     builder.SetCreationTime(creationTime);
     if (logAggregationContext != null)
     {
         builder.SetLogAggregationContext(((LogAggregationContextPBImpl)logAggregationContext
                                           ).GetProto());
     }
     proto = ((YarnSecurityTokenProtos.ContainerTokenIdentifierProto)builder.Build());
 }
Exemple #2
0
 public ApplicationInitEvent(ApplicationId appId, IDictionary <ApplicationAccessType
                                                               , string> acls, LogAggregationContext logAggregationContext)
     : base(appId, ApplicationEventType.InitApplication)
 {
     this.applicationACLs       = acls;
     this.logAggregationContext = logAggregationContext;
 }
Exemple #3
0
 public SchedulerApplicationAttempt(ApplicationAttemptId applicationAttemptId, string
                                    user, Queue queue, ActiveUsersManager activeUsersManager, RMContext rmContext)
 {
     // This pendingRelease is used in work-preserving recovery scenario to keep
     // track of the AM's outstanding release requests. RM on recovery could
     // receive the release request form AM before it receives the container status
     // from NM for recovery. In this case, the to-be-recovered containers reported
     // by NM should not be recovered.
     // Time of the last container scheduled at the current allowed level
     Preconditions.CheckNotNull(rmContext, "RMContext should not be null");
     this.rmContext         = rmContext;
     this.appSchedulingInfo = new AppSchedulingInfo(applicationAttemptId, user, queue,
                                                    activeUsersManager, rmContext.GetEpoch());
     this.queue          = queue;
     this.pendingRelease = new HashSet <ContainerId>();
     this.attemptId      = applicationAttemptId;
     if (rmContext.GetRMApps() != null && rmContext.GetRMApps().Contains(applicationAttemptId
                                                                         .GetApplicationId()))
     {
         ApplicationSubmissionContext appSubmissionContext = rmContext.GetRMApps()[applicationAttemptId
                                                                                   .GetApplicationId()].GetApplicationSubmissionContext();
         if (appSubmissionContext != null)
         {
             unmanagedAM = appSubmissionContext.GetUnmanagedAM();
             this.logAggregationContext = appSubmissionContext.GetLogAggregationContext();
         }
     }
 }
Exemple #4
0
        public virtual void TestLogAggregationContextPassedIntoContainerToken()
        {
            MockRM rm1 = new MockRM(conf);

            rm1.Start();
            MockNM nm1 = rm1.RegisterNode("127.0.0.1:1234", 8000);
            MockNM nm2 = rm1.RegisterNode("127.0.0.1:2345", 8000);

            // LogAggregationContext is set as null
            NUnit.Framework.Assert.IsNull(GetLogAggregationContextFromContainerToken(rm1, nm1
                                                                                     , null));
            // create a not-null LogAggregationContext
            LogAggregationContext logAggregationContext = LogAggregationContext.NewInstance("includePattern"
                                                                                            , "excludePattern", "rolledLogsIncludePattern", "rolledLogsExcludePattern");
            LogAggregationContext returned = GetLogAggregationContextFromContainerToken(rm1,
                                                                                        nm2, logAggregationContext);

            NUnit.Framework.Assert.AreEqual("includePattern", returned.GetIncludePattern());
            NUnit.Framework.Assert.AreEqual("excludePattern", returned.GetExcludePattern());
            NUnit.Framework.Assert.AreEqual("rolledLogsIncludePattern", returned.GetRolledLogsIncludePattern
                                                ());
            NUnit.Framework.Assert.AreEqual("rolledLogsExcludePattern", returned.GetRolledLogsExcludePattern
                                                ());
            rm1.Stop();
        }
Exemple #5
0
 public override Token CreateContainerToken(ContainerId containerId, NodeId nodeId
                                            , string appSubmitter, Resource capability, Priority priority, long createTime,
                                            LogAggregationContext logAggregationContext)
 {
     this._enclosing._enclosing.numRetries++;
     return(base.CreateContainerToken(containerId, nodeId, appSubmitter, capability, priority
                                      , createTime, logAggregationContext));
 }
Exemple #6
0
 /// <exception cref="System.Exception"/>
 public virtual RMApp SubmitApp(int masterMemory, LogAggregationContext logAggregationContext
                                )
 {
     return(SubmitApp(masterMemory, string.Empty, UserGroupInformation.GetCurrentUser(
                          ).GetShortUserName(), null, false, null, base.GetConfig().GetInt(YarnConfiguration
                                                                                           .RmAmMaxAttempts, YarnConfiguration.DefaultRmAmMaxAttempts), null, null, true, false
                      , false, null, 0, logAggregationContext, true));
 }
 public override void SetLogAggregationContext(LogAggregationContext logAggregationContext
                                               )
 {
     MaybeInitBuilder();
     if (logAggregationContext == null)
     {
         builder.ClearLogAggregationContext();
     }
     this.logAggregationContext = logAggregationContext;
 }
Exemple #8
0
 public LogHandlerAppStartedEvent(ApplicationId appId, string user, Credentials credentials
                                  , ContainerLogsRetentionPolicy retentionPolicy, IDictionary <ApplicationAccessType
                                                                                               , string> appAcls, LogAggregationContext logAggregationContext)
     : base(LogHandlerEventType.ApplicationStarted)
 {
     this.applicationId         = appId;
     this.user                  = user;
     this.credentials           = credentials;
     this.retentionPolicy       = retentionPolicy;
     this.appAcls               = appAcls;
     this.logAggregationContext = logAggregationContext;
 }
        /// <exception cref="System.IO.IOException"/>
        public static Token CreateContainerToken(ContainerId cId, long rmIdentifier, NodeId
                                                 nodeId, string user, NMContainerTokenSecretManager containerTokenSecretManager,
                                                 LogAggregationContext logAggregationContext)
        {
            Resource r = BuilderUtils.NewResource(1024, 1);
            ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
                cId, nodeId.ToString(), user, r, Runtime.CurrentTimeMillis() + 100000L, 123, rmIdentifier
                , Priority.NewInstance(0), 0, logAggregationContext);
            Token containerToken = BuilderUtils.NewContainerToken(nodeId, containerTokenSecretManager
                                                                  .RetrievePassword(containerTokenIdentifier), containerTokenIdentifier);

            return(containerToken);
        }
 public override LogAggregationContext GetLogAggregationContext()
 {
     YarnProtos.ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
     if (this.logAggregationContext != null)
     {
         return(this.logAggregationContext);
     }
     // Else via proto
     if (!p.HasLogAggregationContext())
     {
         return(null);
     }
     logAggregationContext = ConvertFromProtoFormat(p.GetLogAggregationContext());
     return(logAggregationContext);
 }
Exemple #11
0
 public LogValue(IList <string> rootLogDirs, ContainerId containerId, string user,
                 LogAggregationContext logAggregationContext, ICollection <string> alreadyUploadedLogFiles
                 , bool appFinished)
 {
     // TODO Maybe add a version string here. Instead of changing the version of
     // the entire k-v format
     this.rootLogDirs = new AList <string>(rootLogDirs);
     this.containerId = containerId;
     this.user        = user;
     // Ensure logs are processed in lexical order
     this.rootLogDirs.Sort();
     this.logAggregationContext   = logAggregationContext;
     this.alreadyUploadedLogFiles = alreadyUploadedLogFiles;
     this.appFinished             = appFinished;
 }
        /// <exception cref="System.Exception"/>
        private StartContainersResponse StartContainer(Context context, ContainerManagerImpl
                                                       cm, ContainerId cid, ContainerLaunchContext clc, LogAggregationContext logAggregationContext
                                                       )
        {
            UserGroupInformation user = UserGroupInformation.CreateRemoteUser(cid.GetApplicationAttemptId
                                                                                  ().ToString());
            StartContainerRequest scReq = StartContainerRequest.NewInstance(clc, TestContainerManager
                                                                            .CreateContainerToken(cid, 0, context.GetNodeId(), user.GetShortUserName(), context
                                                                                                  .GetContainerTokenSecretManager(), logAggregationContext));
            IList <StartContainerRequest> scReqList = new AList <StartContainerRequest>();

            scReqList.AddItem(scReq);
            NMTokenIdentifier nmToken = new NMTokenIdentifier(cid.GetApplicationAttemptId(),
                                                              context.GetNodeId(), user.GetShortUserName(), context.GetNMTokenSecretManager().
                                                              GetCurrentKey().GetKeyId());

            user.AddTokenIdentifier(nmToken);
            return(user.DoAs(new _PrivilegedExceptionAction_264(cm, scReqList)));
        }
        public virtual void TestApplicationRecovery()
        {
            YarnConfiguration conf = new YarnConfiguration();

            conf.SetBoolean(YarnConfiguration.NmRecoveryEnabled, true);
            conf.Set(YarnConfiguration.NmAddress, "localhost:1234");
            conf.SetBoolean(YarnConfiguration.YarnAclEnable, true);
            conf.Set(YarnConfiguration.YarnAdminAcl, "yarn_admin_user");
            NMStateStoreService stateStore = new NMMemoryStateStoreService();

            stateStore.Init(conf);
            stateStore.Start();
            Context context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf
                                                                                          ), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore
                                                        );
            ContainerManagerImpl cm = CreateContainerManager(context);

            cm.Init(conf);
            cm.Start();
            // simulate registration with RM
            MasterKey masterKey = new MasterKeyPBImpl();

            masterKey.SetKeyId(123);
            masterKey.SetBytes(ByteBuffer.Wrap(new byte[] { 123 }));
            context.GetContainerTokenSecretManager().SetMasterKey(masterKey);
            context.GetNMTokenSecretManager().SetMasterKey(masterKey);
            // add an application by starting a container
            string               appUser   = "******";
            string               modUser   = "******";
            string               viewUser  = "******";
            string               enemyUser = "******";
            ApplicationId        appId     = ApplicationId.NewInstance(0, 1);
            ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
            ContainerId          cid       = ContainerId.NewContainerId(attemptId, 1);
            IDictionary <string, LocalResource> localResources = Collections.EmptyMap();
            IDictionary <string, string>        containerEnv   = Sharpen.Collections.EmptyMap();
            IList <string> containerCmds = Sharpen.Collections.EmptyList();
            IDictionary <string, ByteBuffer> serviceData = Sharpen.Collections.EmptyMap();
            Credentials      containerCreds = new Credentials();
            DataOutputBuffer dob            = new DataOutputBuffer();

            containerCreds.WriteTokenStorageToStream(dob);
            ByteBuffer containerTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
            IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
                                                                               , string>();

            acls[ApplicationAccessType.ModifyApp] = modUser;
            acls[ApplicationAccessType.ViewApp]   = viewUser;
            ContainerLaunchContext clc = ContainerLaunchContext.NewInstance(localResources, containerEnv
                                                                            , containerCmds, serviceData, containerTokens, acls);
            // create the logAggregationContext
            LogAggregationContext logAggregationContext = LogAggregationContext.NewInstance("includePattern"
                                                                                            , "excludePattern", "includePatternInRollingAggregation", "excludePatternInRollingAggregation"
                                                                                            );
            StartContainersResponse startResponse = StartContainer(context, cm, cid, clc, logAggregationContext
                                                                   );

            NUnit.Framework.Assert.IsTrue(startResponse.GetFailedRequests().IsEmpty());
            NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count);
            Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                app = context.GetApplications()[appId];
            NUnit.Framework.Assert.IsNotNull(app);
            WaitForAppState(app, ApplicationState.Initing);
            NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                          .CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId));
            NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                           .CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId));
            NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                          .CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId));
            NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                           .CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId));
            // reset container manager and verify app recovered with proper acls
            cm.Stop();
            context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new
                                                NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore);
            cm = CreateContainerManager(context);
            cm.Init(conf);
            cm.Start();
            NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count);
            app = context.GetApplications()[appId];
            NUnit.Framework.Assert.IsNotNull(app);
            // check whether LogAggregationContext is recovered correctly
            LogAggregationContext recovered = ((ApplicationImpl)app).GetLogAggregationContext
                                                  ();

            NUnit.Framework.Assert.IsNotNull(recovered);
            NUnit.Framework.Assert.AreEqual(logAggregationContext.GetIncludePattern(), recovered
                                            .GetIncludePattern());
            NUnit.Framework.Assert.AreEqual(logAggregationContext.GetExcludePattern(), recovered
                                            .GetExcludePattern());
            NUnit.Framework.Assert.AreEqual(logAggregationContext.GetRolledLogsIncludePattern
                                                (), recovered.GetRolledLogsIncludePattern());
            NUnit.Framework.Assert.AreEqual(logAggregationContext.GetRolledLogsExcludePattern
                                                (), recovered.GetRolledLogsExcludePattern());
            WaitForAppState(app, ApplicationState.Initing);
            NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                          .CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId));
            NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                           .CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId));
            NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                          .CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId));
            NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                           .CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId));
            // simulate application completion
            IList <ApplicationId> finishedApps = new AList <ApplicationId>();

            finishedApps.AddItem(appId);
            cm.Handle(new CMgrCompletedAppsEvent(finishedApps, CMgrCompletedAppsEvent.Reason.
                                                 ByResourcemanager));
            WaitForAppState(app, ApplicationState.ApplicationResourcesCleaningup);
            // restart and verify app is marked for finishing
            cm.Stop();
            context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new
                                                NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore);
            cm = CreateContainerManager(context);
            cm.Init(conf);
            cm.Start();
            NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count);
            app = context.GetApplications()[appId];
            NUnit.Framework.Assert.IsNotNull(app);
            WaitForAppState(app, ApplicationState.ApplicationResourcesCleaningup);
            NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                          .CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId));
            NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                           .CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId));
            NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                          .CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId));
            NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                           .CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId));
            // simulate log aggregation completion
            app.Handle(new ApplicationEvent(app.GetAppId(), ApplicationEventType.ApplicationResourcesCleanedup
                                            ));
            NUnit.Framework.Assert.AreEqual(app.GetApplicationState(), ApplicationState.Finished
                                            );
            app.Handle(new ApplicationEvent(app.GetAppId(), ApplicationEventType.ApplicationLogHandlingFinished
                                            ));
            // restart and verify app is no longer present after recovery
            cm.Stop();
            context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new
                                                NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore);
            cm = CreateContainerManager(context);
            cm.Init(conf);
            cm.Start();
            NUnit.Framework.Assert.IsTrue(context.GetApplications().IsEmpty());
            cm.Stop();
        }
 private YarnProtos.LogAggregationContextProto ConvertToProtoFormat(LogAggregationContext
                                                                    t)
 {
     return(((LogAggregationContextPBImpl)t).GetProto());
 }
Exemple #15
0
        /// <exception cref="System.Exception"/>
        public virtual RMApp SubmitApp(int masterMemory, string name, string user, IDictionary
                                       <ApplicationAccessType, string> acls, bool unmanaged, string queue, int maxAppAttempts
                                       , Credentials ts, string appType, bool waitForAccepted, bool keepContainers, bool
                                       isAppIdProvided, ApplicationId applicationId, long attemptFailuresValidityInterval
                                       , LogAggregationContext logAggregationContext, bool cancelTokensWhenComplete)
        {
            ApplicationId             appId  = isAppIdProvided ? applicationId : null;
            ApplicationClientProtocol client = GetClientRMService();

            if (!isAppIdProvided)
            {
                GetNewApplicationResponse resp = client.GetNewApplication(Org.Apache.Hadoop.Yarn.Util.Records
                                                                          .NewRecord <GetNewApplicationRequest>());
                appId = resp.GetApplicationId();
            }
            SubmitApplicationRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <SubmitApplicationRequest
                                                                                          >();
            ApplicationSubmissionContext sub = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <
                ApplicationSubmissionContext>();

            sub.SetKeepContainersAcrossApplicationAttempts(keepContainers);
            sub.SetApplicationId(appId);
            sub.SetApplicationName(name);
            sub.SetMaxAppAttempts(maxAppAttempts);
            if (unmanaged)
            {
                sub.SetUnmanagedAM(true);
            }
            if (queue != null)
            {
                sub.SetQueue(queue);
            }
            sub.SetApplicationType(appType);
            ContainerLaunchContext clc = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext
                                                                                        >();
            Resource capability = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Resource>();

            capability.SetMemory(masterMemory);
            sub.SetResource(capability);
            clc.SetApplicationACLs(acls);
            if (ts != null && UserGroupInformation.IsSecurityEnabled())
            {
                DataOutputBuffer dob = new DataOutputBuffer();
                ts.WriteTokenStorageToStream(dob);
                ByteBuffer securityTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
                clc.SetTokens(securityTokens);
            }
            sub.SetAMContainerSpec(clc);
            sub.SetAttemptFailuresValidityInterval(attemptFailuresValidityInterval);
            if (logAggregationContext != null)
            {
                sub.SetLogAggregationContext(logAggregationContext);
            }
            sub.SetCancelTokensWhenComplete(cancelTokensWhenComplete);
            req.SetApplicationSubmissionContext(sub);
            UserGroupInformation fakeUser = UserGroupInformation.CreateUserForTesting(user, new
                                                                                      string[] { "someGroup" });
            PrivilegedAction <SubmitApplicationResponse> action = new _PrivilegedAction_415().
                                                                  SetClientReq(client, req);

            fakeUser.DoAs(action);
            // make sure app is immediately available after submit
            if (waitForAccepted)
            {
                WaitForState(appId, RMAppState.Accepted);
            }
            RMApp rmApp = GetRMContext().GetRMApps()[appId];

            // unmanaged AM won't go to RMAppAttemptState.SCHEDULED.
            if (waitForAccepted && !unmanaged)
            {
                WaitForState(rmApp.GetCurrentAppAttempt().GetAppAttemptId(), RMAppAttemptState.Scheduled
                             );
            }
            return(rmApp);
        }
Exemple #16
0
        private void InitApp(ApplicationId appId, string user, Credentials credentials, ContainerLogsRetentionPolicy
                             logRetentionPolicy, IDictionary <ApplicationAccessType, string> appAcls, LogAggregationContext
                             logAggregationContext)
        {
            ApplicationEvent eventResponse;

            try
            {
                VerifyAndCreateRemoteLogDir(GetConfig());
                InitAppAggregator(appId, user, credentials, logRetentionPolicy, appAcls, logAggregationContext
                                  );
                eventResponse = new ApplicationEvent(appId, ApplicationEventType.ApplicationLogHandlingInited
                                                     );
            }
            catch (YarnRuntimeException e)
            {
                Log.Warn("Application failed to init aggregation", e);
                eventResponse = new ApplicationEvent(appId, ApplicationEventType.ApplicationLogHandlingFailed
                                                     );
            }
            this.dispatcher.GetEventHandler().Handle(eventResponse);
        }
Exemple #17
0
        protected internal virtual void InitAppAggregator(ApplicationId appId, string user
                                                          , Credentials credentials, ContainerLogsRetentionPolicy logRetentionPolicy, IDictionary
                                                          <ApplicationAccessType, string> appAcls, LogAggregationContext logAggregationContext
                                                          )
        {
            // Get user's FileSystem credentials
            UserGroupInformation userUgi = UserGroupInformation.CreateRemoteUser(user);

            if (credentials != null)
            {
                userUgi.AddCredentials(credentials);
            }
            // New application
            AppLogAggregator appLogAggregator = new AppLogAggregatorImpl(this.dispatcher, this
                                                                         .deletionService, GetConfig(), appId, userUgi, this.nodeId, dirsHandler, GetRemoteNodeLogFileForApp
                                                                             (appId, user), logRetentionPolicy, appAcls, logAggregationContext, this.context,
                                                                         GetLocalFileContext(GetConfig()));

            if (this.appLogAggregators.PutIfAbsent(appId, appLogAggregator) != null)
            {
                throw new YarnRuntimeException("Duplicate initApp for " + appId);
            }
            // wait until check for existing aggregator to create dirs
            YarnRuntimeException appDirException = null;

            try
            {
                // Create the app dir
                CreateAppDir(user, appId, userUgi);
            }
            catch (Exception e)
            {
                appLogAggregator.DisableLogAggregation();
                if (!(e is YarnRuntimeException))
                {
                    appDirException = new YarnRuntimeException(e);
                }
                else
                {
                    appDirException = (YarnRuntimeException)e;
                }
            }
            // TODO Get the user configuration for the list of containers that need log
            // aggregation.
            // Schedule the aggregator.
            Runnable aggregatorWrapper = new _Runnable_381(this, appLogAggregator, appId, userUgi
                                                           );

            this.threadPool.Execute(aggregatorWrapper);
            if (appDirException != null)
            {
                throw appDirException;
            }
        }
        public AppLogAggregatorImpl(Dispatcher dispatcher, DeletionService deletionService
                                    , Configuration conf, ApplicationId appId, UserGroupInformation userUgi, NodeId
                                    nodeId, LocalDirsHandlerService dirsHandler, Path remoteNodeLogFileForApp, ContainerLogsRetentionPolicy
                                    retentionPolicy, IDictionary <ApplicationAccessType, string> appAcls, LogAggregationContext
                                    logAggregationContext, Context context, FileContext lfs)
        {
            // This is temporary solution. The configuration will be deleted once
            // we find a more scalable method to only write a single log file per LRS.
            // This configuration is for debug and test purpose. By setting
            // this configuration as true. We can break the lower bound of
            // NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS.
            // This variable is only for testing
            this.dispatcher                 = dispatcher;
            this.conf                       = conf;
            this.delService                 = deletionService;
            this.appId                      = appId;
            this.applicationId              = ConverterUtils.ToString(appId);
            this.userUgi                    = userUgi;
            this.dirsHandler                = dirsHandler;
            this.remoteNodeLogFileForApp    = remoteNodeLogFileForApp;
            this.remoteNodeTmpLogFileForApp = GetRemoteNodeTmpLogFileForApp();
            this.retentionPolicy            = retentionPolicy;
            this.pendingContainers          = new LinkedBlockingQueue <ContainerId>();
            this.appAcls                    = appAcls;
            this.lfs = lfs;
            this.logAggregationContext = logAggregationContext;
            this.context = context;
            this.nodeId  = nodeId;
            int configuredRentionSize = conf.GetInt(NmLogAggregationNumLogFilesSizePerApp, DefaultNmLogAggregationNumLogFilesSizePerApp
                                                    );

            if (configuredRentionSize <= 0)
            {
                this.retentionSize = DefaultNmLogAggregationNumLogFilesSizePerApp;
            }
            else
            {
                this.retentionSize = configuredRentionSize;
            }
            long configuredRollingMonitorInterval = conf.GetLong(YarnConfiguration.NmLogAggregationRollMonitoringIntervalSeconds
                                                                 , YarnConfiguration.DefaultNmLogAggregationRollMonitoringIntervalSeconds);
            bool debug_mode = conf.GetBoolean(NmLogAggregationDebugEnabled, DefaultNmLogAggregationDebugEnabled
                                              );

            if (configuredRollingMonitorInterval > 0 && configuredRollingMonitorInterval < NmLogAggregationMinRollMonitoringIntervalSeconds)
            {
                if (debug_mode)
                {
                    this.rollingMonitorInterval = configuredRollingMonitorInterval;
                }
                else
                {
                    Log.Warn("rollingMonitorIntervall should be more than or equal to " + NmLogAggregationMinRollMonitoringIntervalSeconds
                             + " seconds. Using " + NmLogAggregationMinRollMonitoringIntervalSeconds + " seconds instead."
                             );
                    this.rollingMonitorInterval = NmLogAggregationMinRollMonitoringIntervalSeconds;
                }
            }
            else
            {
                if (configuredRollingMonitorInterval <= 0)
                {
                    Log.Warn("rollingMonitorInterval is set as " + configuredRollingMonitorInterval +
                             ". " + "The log rolling mornitoring interval is disabled. " + "The logs will be aggregated after this application is finished."
                             );
                }
                else
                {
                    Log.Warn("rollingMonitorInterval is set as " + configuredRollingMonitorInterval +
                             ". " + "The logs will be aggregated every " + configuredRollingMonitorInterval
                             + " seconds");
                }
                this.rollingMonitorInterval = configuredRollingMonitorInterval;
            }
            this.logAggregationInRolling = this.rollingMonitorInterval <= 0 || this.logAggregationContext
                                           == null || this.logAggregationContext.GetRolledLogsIncludePattern() == null ||
                                           this.logAggregationContext.GetRolledLogsIncludePattern().IsEmpty() ? false : true;
        }
Exemple #19
0
        /// <summary>Helper function for creating ContainerTokens</summary>
        /// <param name="containerId"/>
        /// <param name="nodeId"/>
        /// <param name="appSubmitter"/>
        /// <param name="capability"/>
        /// <param name="priority"/>
        /// <param name="createTime"/>
        /// <param name="logAggregationContext"/>
        /// <returns>the container-token</returns>
        public virtual Token CreateContainerToken(ContainerId containerId, NodeId nodeId,
                                                  string appSubmitter, Resource capability, Priority priority, long createTime, LogAggregationContext
                                                  logAggregationContext)
        {
            byte[] password;
            ContainerTokenIdentifier tokenIdentifier;
            long expiryTimeStamp = Runtime.CurrentTimeMillis() + containerTokenExpiryInterval;

            // Lock so that we use the same MasterKey's keyId and its bytes
            this.readLock.Lock();
            try
            {
                tokenIdentifier = new ContainerTokenIdentifier(containerId, nodeId.ToString(), appSubmitter
                                                               , capability, expiryTimeStamp, this.currentMasterKey.GetMasterKey().GetKeyId(),
                                                               ResourceManager.GetClusterTimeStamp(), priority, createTime, logAggregationContext
                                                               );
                password = this.CreatePassword(tokenIdentifier);
            }
            finally
            {
                this.readLock.Unlock();
            }
            return(BuilderUtils.NewContainerToken(nodeId, password, tokenIdentifier));
        }
Exemple #20
0
        /// <exception cref="System.Exception"/>
        private LogAggregationContext GetLogAggregationContextFromContainerToken(MockRM rm1
                                                                                 , MockNM nm1, LogAggregationContext logAggregationContext)
        {
            RMApp  app2 = rm1.SubmitApp(200, logAggregationContext);
            MockAM am2  = MockRM.LaunchAndRegisterAM(app2, rm1, nm1);

            nm1.NodeHeartbeat(true);
            // request a container.
            am2.Allocate("127.0.0.1", 512, 1, new AList <ContainerId>());
            ContainerId containerId = ContainerId.NewContainerId(am2.GetApplicationAttemptId(
                                                                     ), 2);

            rm1.WaitForState(nm1, containerId, RMContainerState.Allocated);
            // acquire the container.
            IList <Container> containers = am2.Allocate(new AList <ResourceRequest>(), new AList
                                                        <ContainerId>()).GetAllocatedContainers();

            NUnit.Framework.Assert.AreEqual(containerId, containers[0].GetId());
            // container token is generated.
            NUnit.Framework.Assert.IsNotNull(containers[0].GetContainerToken());
            ContainerTokenIdentifier token = BuilderUtils.NewContainerTokenIdentifier(containers
                                                                                      [0].GetContainerToken());

            return(token.GetLogAggregationContext());
        }