public NMWebApp(ResourceView resourceView, ApplicationACLsManager aclsManager, LocalDirsHandlerService
dirsHandler)
{
this.resourceView = resourceView;
this.aclsManager = aclsManager;
this.dirsHandler = dirsHandler;
}
protected internal virtual ContainerManagerImpl CreateContainerManager(Context context
, ContainerExecutor exec, DeletionService del, NodeStatusUpdater nodeStatusUpdater
, ApplicationACLsManager aclsManager, LocalDirsHandlerService dirsHandler)
{
return(new ContainerManagerImpl(context, exec, del, nodeStatusUpdater, metrics, aclsManager
, dirsHandler));
}
public _ContainerManagerImpl_438(TestNodeManager2 _enclosing, Context baseArg1, ContainerExecutor
baseArg2, DeletionService baseArg3, NodeStatusUpdater baseArg4, NodeManagerMetrics
baseArg5, ApplicationACLsManager baseArg6, LocalDirsHandlerService baseArg7)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7)
{
this._enclosing = _enclosing;
}
public TestRMAppManager(TestAppManager _enclosing, RMContext context, ClientToAMTokenSecretManagerInRM
clientToAMSecretManager, YarnScheduler scheduler, ApplicationMasterService masterService
, ApplicationACLsManager applicationACLsManager, Configuration conf)
: base(context, scheduler, masterService, applicationACLsManager, conf)
{
this._enclosing = _enclosing;
}
public DummyContainerManager(Context context, ContainerExecutor exec, DeletionService
deletionContext, NodeStatusUpdater nodeStatusUpdater, NodeManagerMetrics metrics
, ApplicationACLsManager applicationACLsManager, LocalDirsHandlerService dirsHandler
)
: base(context, exec, deletionContext, nodeStatusUpdater, metrics, applicationACLsManager
, dirsHandler)
{
}
public WebServer(Context nmContext, ResourceView resourceView, ApplicationACLsManager
aclsManager, LocalDirsHandlerService dirsHandler)
: base(typeof(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.WebServer).FullName
)
{
this.nmContext = nmContext;
this.nmWebApp = new WebServer.NMWebApp(resourceView, aclsManager, dirsHandler);
}
public MyClientRMService(RMContext rmContext, YarnScheduler scheduler, RMAppManager
rmAppManager, ApplicationACLsManager applicationACLsManager, QueueACLsManager queueACLsManager
, RMDelegationTokenSecretManager rmDTSecretManager)
: base(rmContext, scheduler, rmAppManager, applicationACLsManager, queueACLsManager
, rmDTSecretManager)
{
this.rmContext = rmContext;
}
public MyRMAppManager(RMContext context, YarnScheduler scheduler, ApplicationMasterService
masterService, ApplicationACLsManager applicationACLsManager, Configuration conf
)
: base(context, scheduler, masterService, applicationACLsManager, conf)
{
this.conf = conf;
this.rmContext = context;
}
public ApplicationHistoryManagerOnTimelineStore(TimelineDataManager timelineDataManager
, ApplicationACLsManager aclsManager)
: base(typeof(Org.Apache.Hadoop.Yarn.Server.Applicationhistoryservice.ApplicationHistoryManagerOnTimelineStore
).FullName)
{
this.timelineDataManager = timelineDataManager;
this.aclsManager = aclsManager;
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
conf.SetBoolean(Dispatcher.DispatcherExitOnErrorKey, true);
rmWorkPreservingRestartEnabled = conf.GetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled
, YarnConfiguration.DefaultRmWorkPreservingRecoveryEnabled);
InitAndStartRecoveryStore(conf);
NMContainerTokenSecretManager containerTokenSecretManager = new NMContainerTokenSecretManager
(conf, nmStore);
NMTokenSecretManagerInNM nmTokenSecretManager = new NMTokenSecretManagerInNM(nmStore
);
RecoverTokens(nmTokenSecretManager, containerTokenSecretManager);
this.aclsManager = new ApplicationACLsManager(conf);
ContainerExecutor exec = ReflectionUtils.NewInstance(conf.GetClass <ContainerExecutor
>(YarnConfiguration.NmContainerExecutor, typeof(DefaultContainerExecutor)), conf
);
try
{
exec.Init();
}
catch (IOException e)
{
throw new YarnRuntimeException("Failed to initialize container executor", e);
}
DeletionService del = CreateDeletionService(exec);
AddService(del);
// NodeManager level dispatcher
this.dispatcher = new AsyncDispatcher();
nodeHealthChecker = new NodeHealthCheckerService();
AddService(nodeHealthChecker);
dirsHandler = nodeHealthChecker.GetDiskHandler();
this.context = CreateNMContext(containerTokenSecretManager, nmTokenSecretManager,
nmStore);
nodeStatusUpdater = CreateNodeStatusUpdater(context, dispatcher, nodeHealthChecker
);
NodeResourceMonitor nodeResourceMonitor = CreateNodeResourceMonitor();
AddService(nodeResourceMonitor);
containerManager = CreateContainerManager(context, exec, del, nodeStatusUpdater,
this.aclsManager, dirsHandler);
AddService(containerManager);
((NodeManager.NMContext)context).SetContainerManager(containerManager);
WebServer webServer = CreateWebServer(context, containerManager.GetContainersMonitor
(), this.aclsManager, dirsHandler);
AddService(webServer);
((NodeManager.NMContext)context).SetWebServer(webServer);
dispatcher.Register(typeof(ContainerManagerEventType), containerManager);
dispatcher.Register(typeof(NodeManagerEventType), this);
AddService(dispatcher);
DefaultMetricsSystem.Initialize("NodeManager");
// StatusUpdater should be added last so that it get started last
// so that we make sure everything is up before registering with RM.
AddService(nodeStatusUpdater);
base.ServiceInit(conf);
}
public _ContainerManagerImpl_319(LogHandler logHandler, ResourceLocalizationService
rsrcSrv, ContainersLauncher launcher, Context baseArg1, ContainerExecutor baseArg2
, DeletionService baseArg3, NodeStatusUpdater baseArg4, NodeManagerMetrics baseArg5
, ApplicationACLsManager baseArg6, LocalDirsHandlerService baseArg7)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7)
{
this.logHandler = logHandler;
this.rsrcSrv = rsrcSrv;
this.launcher = launcher;
}
public virtual void Setup()
{
// Only test the ACLs of the generic history
TimelineACLsManager aclsManager = new TimelineACLsManager(new YarnConfiguration()
);
TimelineDataManager dataManager = new TimelineDataManager(store, aclsManager);
ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
historyManager = new ApplicationHistoryManagerOnTimelineStore(dataManager, appAclsManager
);
historyManager.Init(conf);
historyManager.Start();
}
/// <exception cref="System.IO.IOException"/>
public static ResourceManager MockRm(RMContext rmContext)
{
ResourceManager rm = Org.Mockito.Mockito.Mock <ResourceManager>();
ResourceScheduler rs = MockCapacityScheduler();
ApplicationACLsManager aclMgr = MockAppACLsManager();
ClientRMService clientRMService = MockClientRMService(rmContext);
Org.Mockito.Mockito.When(rm.GetResourceScheduler()).ThenReturn(rs);
Org.Mockito.Mockito.When(rm.GetRMContext()).ThenReturn(rmContext);
Org.Mockito.Mockito.When(rm.GetApplicationACLsManager()).ThenReturn(aclMgr);
Org.Mockito.Mockito.When(rm.GetClientRMService()).ThenReturn(clientRMService);
return(rm);
}
public NMContext(NMContainerTokenSecretManager containerTokenSecretManager, NMTokenSecretManagerInNM
nmTokenSecretManager, LocalDirsHandlerService dirsHandler, ApplicationACLsManager
aclsManager, NMStateStoreService stateStore)
{
this.containerTokenSecretManager = containerTokenSecretManager;
this.nmTokenSecretManager = nmTokenSecretManager;
this.dirsHandler = dirsHandler;
this.aclsManager = aclsManager;
this.nodeHealthStatus.SetIsNodeHealthy(true);
this.nodeHealthStatus.SetHealthReport("Healthy");
this.nodeHealthStatus.SetLastHealthReportTime(Runtime.CurrentTimeMillis());
this.stateStore = stateStore;
}
public ApplicationImpl(Dispatcher dispatcher, string user, ApplicationId appId, Credentials
credentials, Context context)
{
this.dispatcher = dispatcher;
this.user = user;
this.appId = appId;
this.credentials = credentials;
this.aclsManager = context.GetApplicationACLsManager();
this.context = context;
ReentrantReadWriteLock Lock = new ReentrantReadWriteLock();
readLock = Lock.ReadLock();
writeLock = Lock.WriteLock();
stateMachine = stateMachineFactory.Make(this);
}
public static void Setup()
{
Configuration conf = new YarnConfiguration();
TimelineStore store = TestApplicationHistoryManagerOnTimelineStore.CreateStore(MaxApps
);
TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
dataManager = new TimelineDataManager(store, aclsManager);
ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
ApplicationHistoryManagerOnTimelineStore historyManager = new ApplicationHistoryManagerOnTimelineStore
(dataManager, appAclsManager);
historyManager.Init(conf);
historyManager.Start();
clientService = new ApplicationHistoryClientService(historyManager);
}
public RMAppManager(RMContext context, YarnScheduler scheduler, ApplicationMasterService
masterService, ApplicationACLsManager applicationACLsManager, Configuration conf
)
{
this.rmContext = context;
this.scheduler = scheduler;
this.masterService = masterService;
this.applicationACLsManager = applicationACLsManager;
this.conf = conf;
this.maxCompletedAppsInMemory = conf.GetInt(YarnConfiguration.RmMaxCompletedApplications
, YarnConfiguration.DefaultRmMaxCompletedApplications);
this.maxCompletedAppsInStateStore = conf.GetInt(YarnConfiguration.RmStateStoreMaxCompletedApplications
, YarnConfiguration.DefaultRmStateStoreMaxCompletedApplications);
if (this.maxCompletedAppsInStateStore > this.maxCompletedAppsInMemory)
{
this.maxCompletedAppsInStateStore = this.maxCompletedAppsInMemory;
}
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
// init timeline services first
timelineStore = CreateTimelineStore(conf);
AddIfService(timelineStore);
secretManagerService = CreateTimelineDelegationTokenSecretManagerService(conf);
AddService(secretManagerService);
timelineDataManager = CreateTimelineDataManager(conf);
AddService(timelineDataManager);
// init generic history service afterwards
aclsManager = CreateApplicationACLsManager(conf);
historyManager = CreateApplicationHistoryManager(conf);
ahsClientService = CreateApplicationHistoryClientService(historyManager);
AddService(ahsClientService);
AddService((Org.Apache.Hadoop.Service.Service)historyManager);
DefaultMetricsSystem.Initialize("ApplicationHistoryServer");
JvmMetrics.InitSingleton("ApplicationHistoryServer", null);
base.ServiceInit(conf);
}
public static void SetupClass()
{
Configuration conf = new YarnConfiguration();
TimelineStore store = TestApplicationHistoryManagerOnTimelineStore.CreateStore(MaxApps
);
TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
TimelineDataManager dataManager = new TimelineDataManager(store, aclsManager);
conf.SetBoolean(YarnConfiguration.YarnAclEnable, true);
conf.Set(YarnConfiguration.YarnAdminAcl, "foo");
ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
ApplicationHistoryManagerOnTimelineStore historyManager = new ApplicationHistoryManagerOnTimelineStore
(dataManager, appAclsManager);
historyManager.Init(conf);
historyClientService = new _ApplicationHistoryClientService_98(historyManager);
// Do Nothing
historyClientService.Init(conf);
historyClientService.Start();
}
public _ClientRMService_541(RMContext baseArg1, YarnScheduler baseArg2, RMAppManager
baseArg3, ApplicationACLsManager baseArg4, QueueACLsManager baseArg5, RMDelegationTokenSecretManager
baseArg6)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6)
{
}
protected internal virtual WebServer CreateWebServer(Context nmContext, ResourceView
resourceView, ApplicationACLsManager aclsManager, LocalDirsHandlerService dirsHandler
)
{
return(new WebServer(nmContext, resourceView, aclsManager, dirsHandler));
}
public CustomedClientRMService(MiniYARNClusterForHATesting _enclosing, RMContext
rmContext, YarnScheduler scheduler, RMAppManager rmAppManager, ApplicationACLsManager
applicationACLsManager, QueueACLsManager queueACLsManager, RMDelegationTokenSecretManager
rmDTSecretManager)
: base(rmContext, scheduler, rmAppManager, applicationACLsManager, queueACLsManager
, rmDTSecretManager)
{
this._enclosing = _enclosing;
}
protected internal override void Render(HtmlBlock.Block html)
{
ContainerId containerId = VerifyAndGetContainerId(html);
NodeId nodeId = VerifyAndGetNodeId(html);
string appOwner = VerifyAndGetAppOwner(html);
AggregatedLogsBlock.LogLimits logLimits = VerifyAndGetLogLimits(html);
if (containerId == null || nodeId == null || appOwner == null || appOwner.IsEmpty
() || logLimits == null)
{
return;
}
ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId
();
string logEntity = $(YarnWebParams.EntityString);
if (logEntity == null || logEntity.IsEmpty())
{
logEntity = containerId.ToString();
}
if (!conf.GetBoolean(YarnConfiguration.LogAggregationEnabled, YarnConfiguration.DefaultLogAggregationEnabled
))
{
html.H1().("Aggregation is not enabled. Try the nodemanager at " + nodeId).();
return;
}
Path remoteRootLogDir = new Path(conf.Get(YarnConfiguration.NmRemoteAppLogDir, YarnConfiguration
.DefaultNmRemoteAppLogDir));
Path remoteAppDir = LogAggregationUtils.GetRemoteAppLogDir(remoteRootLogDir, applicationId
, appOwner, LogAggregationUtils.GetRemoteNodeLogDirSuffix(conf));
RemoteIterator <FileStatus> nodeFiles;
try
{
Path qualifiedLogDir = FileContext.GetFileContext(conf).MakeQualified(remoteAppDir
);
nodeFiles = FileContext.GetFileContext(qualifiedLogDir.ToUri(), conf).ListStatus(
remoteAppDir);
}
catch (FileNotFoundException)
{
html.H1().("Logs not available for " + logEntity + ". Aggregation may not be complete, "
+ "Check back later or try the nodemanager at " + nodeId).();
return;
}
catch (Exception)
{
html.H1().("Error getting logs at " + nodeId).();
return;
}
bool foundLog = false;
string desiredLogType = $(YarnWebParams.ContainerLogType);
try
{
while (nodeFiles.HasNext())
{
AggregatedLogFormat.LogReader reader = null;
try
{
FileStatus thisNodeFile = nodeFiles.Next();
if (!thisNodeFile.GetPath().GetName().Contains(LogAggregationUtils.GetNodeString(
nodeId)) || thisNodeFile.GetPath().GetName().EndsWith(LogAggregationUtils.TmpFileSuffix
))
{
continue;
}
long logUploadedTime = thisNodeFile.GetModificationTime();
reader = new AggregatedLogFormat.LogReader(conf, thisNodeFile.GetPath());
string owner = null;
IDictionary <ApplicationAccessType, string> appAcls = null;
try
{
owner = reader.GetApplicationOwner();
appAcls = reader.GetApplicationAcls();
}
catch (IOException e)
{
Log.Error("Error getting logs for " + logEntity, e);
continue;
}
ApplicationACLsManager aclsManager = new ApplicationACLsManager(conf);
aclsManager.AddApplication(applicationId, appAcls);
string remoteUser = Request().GetRemoteUser();
UserGroupInformation callerUGI = null;
if (remoteUser != null)
{
callerUGI = UserGroupInformation.CreateRemoteUser(remoteUser);
}
if (callerUGI != null && !aclsManager.CheckAccess(callerUGI, ApplicationAccessType
.ViewApp, owner, applicationId))
{
html.H1().("User [" + remoteUser + "] is not authorized to view the logs for " +
logEntity + " in log file [" + thisNodeFile.GetPath().GetName() + "]").();
Log.Error("User [" + remoteUser + "] is not authorized to view the logs for " + logEntity
);
continue;
}
AggregatedLogFormat.ContainerLogsReader logReader = reader.GetContainerLogsReader
(containerId);
if (logReader == null)
{
continue;
}
foundLog = ReadContainerLogs(html, logReader, logLimits, desiredLogType, logUploadedTime
);
}
catch (IOException ex)
{
Log.Error("Error getting logs for " + logEntity, ex);
continue;
}
finally
{
if (reader != null)
{
reader.Close();
}
}
}
if (!foundLog)
{
if (desiredLogType.IsEmpty())
{
html.H1("No logs available for container " + containerId.ToString());
}
else
{
html.H1("Unable to locate '" + desiredLogType + "' log for container " + containerId
.ToString());
}
}
}
catch (IOException e)
{
html.H1().("Error getting logs for " + logEntity).();
Log.Error("Error getting logs for " + logEntity, e);
}
}
public _ContainerManagerImpl_191(Context baseArg1, ContainerExecutor baseArg2, DeletionService
baseArg3, NodeStatusUpdater baseArg4, NodeManagerMetrics baseArg5, ApplicationACLsManager
baseArg6, LocalDirsHandlerService baseArg7)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7)
{
}
public _NMContext_108(NMContainerTokenSecretManager baseArg1, NMTokenSecretManagerInNM
baseArg2, LocalDirsHandlerService baseArg3, ApplicationACLsManager baseArg4, NMStateStoreService
baseArg5)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5)
{
}
protected internal override ContainerManagerImpl CreateContainerManager(Context context
, ContainerExecutor exec, DeletionService del, NodeStatusUpdater nodeStatusUpdater
, ApplicationACLsManager aclsManager, LocalDirsHandlerService dirsHandler)
{
return(new _ContainerManagerImpl_438(this, context, exec, del, nodeStatusUpdater,
this.metrics, aclsManager, dirsHandler));
}
