public NMWebApp(ResourceView resourceView, ApplicationACLsManager aclsManager, LocalDirsHandlerService dirsHandler) { this.resourceView = resourceView; this.aclsManager = aclsManager; this.dirsHandler = dirsHandler; }
protected internal virtual ContainerManagerImpl CreateContainerManager(Context context , ContainerExecutor exec, DeletionService del, NodeStatusUpdater nodeStatusUpdater , ApplicationACLsManager aclsManager, LocalDirsHandlerService dirsHandler) { return(new ContainerManagerImpl(context, exec, del, nodeStatusUpdater, metrics, aclsManager , dirsHandler)); }
public _ContainerManagerImpl_438(TestNodeManager2 _enclosing, Context baseArg1, ContainerExecutor baseArg2, DeletionService baseArg3, NodeStatusUpdater baseArg4, NodeManagerMetrics baseArg5, ApplicationACLsManager baseArg6, LocalDirsHandlerService baseArg7) : base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7) { this._enclosing = _enclosing; }
public TestRMAppManager(TestAppManager _enclosing, RMContext context, ClientToAMTokenSecretManagerInRM clientToAMSecretManager, YarnScheduler scheduler, ApplicationMasterService masterService , ApplicationACLsManager applicationACLsManager, Configuration conf) : base(context, scheduler, masterService, applicationACLsManager, conf) { this._enclosing = _enclosing; }
public DummyContainerManager(Context context, ContainerExecutor exec, DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater, NodeManagerMetrics metrics , ApplicationACLsManager applicationACLsManager, LocalDirsHandlerService dirsHandler ) : base(context, exec, deletionContext, nodeStatusUpdater, metrics, applicationACLsManager , dirsHandler) { }
public WebServer(Context nmContext, ResourceView resourceView, ApplicationACLsManager aclsManager, LocalDirsHandlerService dirsHandler) : base(typeof(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.WebServer).FullName ) { this.nmContext = nmContext; this.nmWebApp = new WebServer.NMWebApp(resourceView, aclsManager, dirsHandler); }
public MyClientRMService(RMContext rmContext, YarnScheduler scheduler, RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager, QueueACLsManager queueACLsManager , RMDelegationTokenSecretManager rmDTSecretManager) : base(rmContext, scheduler, rmAppManager, applicationACLsManager, queueACLsManager , rmDTSecretManager) { this.rmContext = rmContext; }
public MyRMAppManager(RMContext context, YarnScheduler scheduler, ApplicationMasterService masterService, ApplicationACLsManager applicationACLsManager, Configuration conf ) : base(context, scheduler, masterService, applicationACLsManager, conf) { this.conf = conf; this.rmContext = context; }
public ApplicationHistoryManagerOnTimelineStore(TimelineDataManager timelineDataManager , ApplicationACLsManager aclsManager) : base(typeof(Org.Apache.Hadoop.Yarn.Server.Applicationhistoryservice.ApplicationHistoryManagerOnTimelineStore ).FullName) { this.timelineDataManager = timelineDataManager; this.aclsManager = aclsManager; }
/// <exception cref="System.Exception"/> protected override void ServiceInit(Configuration conf) { conf.SetBoolean(Dispatcher.DispatcherExitOnErrorKey, true); rmWorkPreservingRestartEnabled = conf.GetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled , YarnConfiguration.DefaultRmWorkPreservingRecoveryEnabled); InitAndStartRecoveryStore(conf); NMContainerTokenSecretManager containerTokenSecretManager = new NMContainerTokenSecretManager (conf, nmStore); NMTokenSecretManagerInNM nmTokenSecretManager = new NMTokenSecretManagerInNM(nmStore ); RecoverTokens(nmTokenSecretManager, containerTokenSecretManager); this.aclsManager = new ApplicationACLsManager(conf); ContainerExecutor exec = ReflectionUtils.NewInstance(conf.GetClass <ContainerExecutor >(YarnConfiguration.NmContainerExecutor, typeof(DefaultContainerExecutor)), conf ); try { exec.Init(); } catch (IOException e) { throw new YarnRuntimeException("Failed to initialize container executor", e); } DeletionService del = CreateDeletionService(exec); AddService(del); // NodeManager level dispatcher this.dispatcher = new AsyncDispatcher(); nodeHealthChecker = new NodeHealthCheckerService(); AddService(nodeHealthChecker); dirsHandler = nodeHealthChecker.GetDiskHandler(); this.context = CreateNMContext(containerTokenSecretManager, nmTokenSecretManager, nmStore); nodeStatusUpdater = CreateNodeStatusUpdater(context, dispatcher, nodeHealthChecker ); NodeResourceMonitor nodeResourceMonitor = CreateNodeResourceMonitor(); AddService(nodeResourceMonitor); containerManager = CreateContainerManager(context, exec, del, nodeStatusUpdater, this.aclsManager, dirsHandler); AddService(containerManager); ((NodeManager.NMContext)context).SetContainerManager(containerManager); WebServer webServer = CreateWebServer(context, containerManager.GetContainersMonitor (), this.aclsManager, dirsHandler); AddService(webServer); ((NodeManager.NMContext)context).SetWebServer(webServer); dispatcher.Register(typeof(ContainerManagerEventType), containerManager); dispatcher.Register(typeof(NodeManagerEventType), this); AddService(dispatcher); DefaultMetricsSystem.Initialize("NodeManager"); // StatusUpdater should be added last so that it get started last // so that we make sure everything is up before registering with RM. AddService(nodeStatusUpdater); base.ServiceInit(conf); }
public _ContainerManagerImpl_319(LogHandler logHandler, ResourceLocalizationService rsrcSrv, ContainersLauncher launcher, Context baseArg1, ContainerExecutor baseArg2 , DeletionService baseArg3, NodeStatusUpdater baseArg4, NodeManagerMetrics baseArg5 , ApplicationACLsManager baseArg6, LocalDirsHandlerService baseArg7) : base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7) { this.logHandler = logHandler; this.rsrcSrv = rsrcSrv; this.launcher = launcher; }
public virtual void Setup() { // Only test the ACLs of the generic history TimelineACLsManager aclsManager = new TimelineACLsManager(new YarnConfiguration() ); TimelineDataManager dataManager = new TimelineDataManager(store, aclsManager); ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf); historyManager = new ApplicationHistoryManagerOnTimelineStore(dataManager, appAclsManager ); historyManager.Init(conf); historyManager.Start(); }
/// <exception cref="System.IO.IOException"/> public static ResourceManager MockRm(RMContext rmContext) { ResourceManager rm = Org.Mockito.Mockito.Mock <ResourceManager>(); ResourceScheduler rs = MockCapacityScheduler(); ApplicationACLsManager aclMgr = MockAppACLsManager(); ClientRMService clientRMService = MockClientRMService(rmContext); Org.Mockito.Mockito.When(rm.GetResourceScheduler()).ThenReturn(rs); Org.Mockito.Mockito.When(rm.GetRMContext()).ThenReturn(rmContext); Org.Mockito.Mockito.When(rm.GetApplicationACLsManager()).ThenReturn(aclMgr); Org.Mockito.Mockito.When(rm.GetClientRMService()).ThenReturn(clientRMService); return(rm); }
public NMContext(NMContainerTokenSecretManager containerTokenSecretManager, NMTokenSecretManagerInNM nmTokenSecretManager, LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager, NMStateStoreService stateStore) { this.containerTokenSecretManager = containerTokenSecretManager; this.nmTokenSecretManager = nmTokenSecretManager; this.dirsHandler = dirsHandler; this.aclsManager = aclsManager; this.nodeHealthStatus.SetIsNodeHealthy(true); this.nodeHealthStatus.SetHealthReport("Healthy"); this.nodeHealthStatus.SetLastHealthReportTime(Runtime.CurrentTimeMillis()); this.stateStore = stateStore; }
public ApplicationImpl(Dispatcher dispatcher, string user, ApplicationId appId, Credentials credentials, Context context) { this.dispatcher = dispatcher; this.user = user; this.appId = appId; this.credentials = credentials; this.aclsManager = context.GetApplicationACLsManager(); this.context = context; ReentrantReadWriteLock Lock = new ReentrantReadWriteLock(); readLock = Lock.ReadLock(); writeLock = Lock.WriteLock(); stateMachine = stateMachineFactory.Make(this); }
public static void Setup() { Configuration conf = new YarnConfiguration(); TimelineStore store = TestApplicationHistoryManagerOnTimelineStore.CreateStore(MaxApps ); TimelineACLsManager aclsManager = new TimelineACLsManager(conf); dataManager = new TimelineDataManager(store, aclsManager); ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf); ApplicationHistoryManagerOnTimelineStore historyManager = new ApplicationHistoryManagerOnTimelineStore (dataManager, appAclsManager); historyManager.Init(conf); historyManager.Start(); clientService = new ApplicationHistoryClientService(historyManager); }
public RMAppManager(RMContext context, YarnScheduler scheduler, ApplicationMasterService masterService, ApplicationACLsManager applicationACLsManager, Configuration conf ) { this.rmContext = context; this.scheduler = scheduler; this.masterService = masterService; this.applicationACLsManager = applicationACLsManager; this.conf = conf; this.maxCompletedAppsInMemory = conf.GetInt(YarnConfiguration.RmMaxCompletedApplications , YarnConfiguration.DefaultRmMaxCompletedApplications); this.maxCompletedAppsInStateStore = conf.GetInt(YarnConfiguration.RmStateStoreMaxCompletedApplications , YarnConfiguration.DefaultRmStateStoreMaxCompletedApplications); if (this.maxCompletedAppsInStateStore > this.maxCompletedAppsInMemory) { this.maxCompletedAppsInStateStore = this.maxCompletedAppsInMemory; } }
/// <exception cref="System.Exception"/> protected override void ServiceInit(Configuration conf) { // init timeline services first timelineStore = CreateTimelineStore(conf); AddIfService(timelineStore); secretManagerService = CreateTimelineDelegationTokenSecretManagerService(conf); AddService(secretManagerService); timelineDataManager = CreateTimelineDataManager(conf); AddService(timelineDataManager); // init generic history service afterwards aclsManager = CreateApplicationACLsManager(conf); historyManager = CreateApplicationHistoryManager(conf); ahsClientService = CreateApplicationHistoryClientService(historyManager); AddService(ahsClientService); AddService((Org.Apache.Hadoop.Service.Service)historyManager); DefaultMetricsSystem.Initialize("ApplicationHistoryServer"); JvmMetrics.InitSingleton("ApplicationHistoryServer", null); base.ServiceInit(conf); }
public static void SetupClass() { Configuration conf = new YarnConfiguration(); TimelineStore store = TestApplicationHistoryManagerOnTimelineStore.CreateStore(MaxApps ); TimelineACLsManager aclsManager = new TimelineACLsManager(conf); TimelineDataManager dataManager = new TimelineDataManager(store, aclsManager); conf.SetBoolean(YarnConfiguration.YarnAclEnable, true); conf.Set(YarnConfiguration.YarnAdminAcl, "foo"); ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf); ApplicationHistoryManagerOnTimelineStore historyManager = new ApplicationHistoryManagerOnTimelineStore (dataManager, appAclsManager); historyManager.Init(conf); historyClientService = new _ApplicationHistoryClientService_98(historyManager); // Do Nothing historyClientService.Init(conf); historyClientService.Start(); }
public _ClientRMService_541(RMContext baseArg1, YarnScheduler baseArg2, RMAppManager baseArg3, ApplicationACLsManager baseArg4, QueueACLsManager baseArg5, RMDelegationTokenSecretManager baseArg6) : base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6) { }
protected internal virtual WebServer CreateWebServer(Context nmContext, ResourceView resourceView, ApplicationACLsManager aclsManager, LocalDirsHandlerService dirsHandler ) { return(new WebServer(nmContext, resourceView, aclsManager, dirsHandler)); }
public CustomedClientRMService(MiniYARNClusterForHATesting _enclosing, RMContext rmContext, YarnScheduler scheduler, RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager, QueueACLsManager queueACLsManager, RMDelegationTokenSecretManager rmDTSecretManager) : base(rmContext, scheduler, rmAppManager, applicationACLsManager, queueACLsManager , rmDTSecretManager) { this._enclosing = _enclosing; }
protected internal override void Render(HtmlBlock.Block html) { ContainerId containerId = VerifyAndGetContainerId(html); NodeId nodeId = VerifyAndGetNodeId(html); string appOwner = VerifyAndGetAppOwner(html); AggregatedLogsBlock.LogLimits logLimits = VerifyAndGetLogLimits(html); if (containerId == null || nodeId == null || appOwner == null || appOwner.IsEmpty () || logLimits == null) { return; } ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId (); string logEntity = $(YarnWebParams.EntityString); if (logEntity == null || logEntity.IsEmpty()) { logEntity = containerId.ToString(); } if (!conf.GetBoolean(YarnConfiguration.LogAggregationEnabled, YarnConfiguration.DefaultLogAggregationEnabled )) { html.H1().("Aggregation is not enabled. Try the nodemanager at " + nodeId).(); return; } Path remoteRootLogDir = new Path(conf.Get(YarnConfiguration.NmRemoteAppLogDir, YarnConfiguration .DefaultNmRemoteAppLogDir)); Path remoteAppDir = LogAggregationUtils.GetRemoteAppLogDir(remoteRootLogDir, applicationId , appOwner, LogAggregationUtils.GetRemoteNodeLogDirSuffix(conf)); RemoteIterator <FileStatus> nodeFiles; try { Path qualifiedLogDir = FileContext.GetFileContext(conf).MakeQualified(remoteAppDir ); nodeFiles = FileContext.GetFileContext(qualifiedLogDir.ToUri(), conf).ListStatus( remoteAppDir); } catch (FileNotFoundException) { html.H1().("Logs not available for " + logEntity + ". Aggregation may not be complete, " + "Check back later or try the nodemanager at " + nodeId).(); return; } catch (Exception) { html.H1().("Error getting logs at " + nodeId).(); return; } bool foundLog = false; string desiredLogType = $(YarnWebParams.ContainerLogType); try { while (nodeFiles.HasNext()) { AggregatedLogFormat.LogReader reader = null; try { FileStatus thisNodeFile = nodeFiles.Next(); if (!thisNodeFile.GetPath().GetName().Contains(LogAggregationUtils.GetNodeString( nodeId)) || thisNodeFile.GetPath().GetName().EndsWith(LogAggregationUtils.TmpFileSuffix )) { continue; } long logUploadedTime = thisNodeFile.GetModificationTime(); reader = new AggregatedLogFormat.LogReader(conf, thisNodeFile.GetPath()); string owner = null; IDictionary <ApplicationAccessType, string> appAcls = null; try { owner = reader.GetApplicationOwner(); appAcls = reader.GetApplicationAcls(); } catch (IOException e) { Log.Error("Error getting logs for " + logEntity, e); continue; } ApplicationACLsManager aclsManager = new ApplicationACLsManager(conf); aclsManager.AddApplication(applicationId, appAcls); string remoteUser = Request().GetRemoteUser(); UserGroupInformation callerUGI = null; if (remoteUser != null) { callerUGI = UserGroupInformation.CreateRemoteUser(remoteUser); } if (callerUGI != null && !aclsManager.CheckAccess(callerUGI, ApplicationAccessType .ViewApp, owner, applicationId)) { html.H1().("User [" + remoteUser + "] is not authorized to view the logs for " + logEntity + " in log file [" + thisNodeFile.GetPath().GetName() + "]").(); Log.Error("User [" + remoteUser + "] is not authorized to view the logs for " + logEntity ); continue; } AggregatedLogFormat.ContainerLogsReader logReader = reader.GetContainerLogsReader (containerId); if (logReader == null) { continue; } foundLog = ReadContainerLogs(html, logReader, logLimits, desiredLogType, logUploadedTime ); } catch (IOException ex) { Log.Error("Error getting logs for " + logEntity, ex); continue; } finally { if (reader != null) { reader.Close(); } } } if (!foundLog) { if (desiredLogType.IsEmpty()) { html.H1("No logs available for container " + containerId.ToString()); } else { html.H1("Unable to locate '" + desiredLogType + "' log for container " + containerId .ToString()); } } } catch (IOException e) { html.H1().("Error getting logs for " + logEntity).(); Log.Error("Error getting logs for " + logEntity, e); } }
public _ContainerManagerImpl_191(Context baseArg1, ContainerExecutor baseArg2, DeletionService baseArg3, NodeStatusUpdater baseArg4, NodeManagerMetrics baseArg5, ApplicationACLsManager baseArg6, LocalDirsHandlerService baseArg7) : base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7) { }
public _NMContext_108(NMContainerTokenSecretManager baseArg1, NMTokenSecretManagerInNM baseArg2, LocalDirsHandlerService baseArg3, ApplicationACLsManager baseArg4, NMStateStoreService baseArg5) : base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5) { }
protected internal override ContainerManagerImpl CreateContainerManager(Context context , ContainerExecutor exec, DeletionService del, NodeStatusUpdater nodeStatusUpdater , ApplicationACLsManager aclsManager, LocalDirsHandlerService dirsHandler) { return(new _ContainerManagerImpl_438(this, context, exec, del, nodeStatusUpdater, this.metrics, aclsManager, dirsHandler)); }