public virtual void TestNMWebApp() { Context nmContext = new NodeManager.NMContext(null, null, null, null, null); ResourceView resourceView = new _ResourceView_147(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLocalDirs, testRootDir.GetAbsolutePath()); conf.Set(YarnConfiguration.NmLogDirs, testLogDir.GetAbsolutePath()); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); healthChecker.Init(conf); LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler(); WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager (conf), dirsHandler); server.Init(conf); server.Start(); // Add an application and the corresponding containers RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf); Dispatcher dispatcher = new AsyncDispatcher(); string user = "******"; long clusterTimeStamp = 1234; ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp , 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user); Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId); nmContext.GetApplications()[appId] = app; ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId , 0); ContainerId container2 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId , 1); NodeManagerMetrics metrics = Org.Mockito.Mockito.Mock <NodeManagerMetrics>(); NMStateStoreService stateStore = new NMNullStateStoreService(); foreach (ContainerId containerId in new ContainerId[] { container1, container2 }) { // TODO: Use builder utils ContainerLaunchContext launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); long currentTime = Runtime.CurrentTimeMillis(); Token containerToken = BuilderUtils.NewContainerToken(containerId, "127.0.0.1", 1234 , user, BuilderUtils.NewResource(1024, 1), currentTime + 10000L, 123, Sharpen.Runtime.GetBytesForString ("password"), currentTime); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = new _ContainerImpl_214(conf, dispatcher, stateStore, launchContext, null, metrics , BuilderUtils.NewContainerTokenIdentifier(containerToken)); nmContext.GetContainers()[containerId] = container; //TODO: Gross hack. Fix in code. ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId (); nmContext.GetApplications()[applicationId].GetContainers()[containerId] = container; WriteContainerLogs(nmContext, containerId, dirsHandler); } }
/// <summary>Finds the log file with the given filename for the given container.</summary> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public static FilePath GetContainerLogFile(ContainerId containerId, string fileName , string remoteUser, Context context) { Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = context.GetContainers()[containerId]; Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application application = GetApplicationForContainer(containerId, context); CheckAccess(remoteUser, application, context); if (container != null) { CheckState(container.GetContainerState()); } try { LocalDirsHandlerService dirsHandler = context.GetLocalDirsHandler(); string relativeContainerLogDir = ContainerLaunch.GetRelativeContainerLogDir(application .GetAppId().ToString(), containerId.ToString()); Path logPath = dirsHandler.GetLogPathToRead(relativeContainerLogDir + Path.Separator + fileName); URI logPathURI = new FilePath(logPath.ToString()).ToURI(); FilePath logFile = new FilePath(logPathURI.GetPath()); return(logFile); } catch (IOException e) { Log.Warn("Failed to find log file", e); throw new NotFoundException("Cannot find this log on the local disk."); } }
/// <exception cref="System.IO.IOException"/> private Dictionary <string, string> AddAppContainers(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app) { Dispatcher dispatcher = new AsyncDispatcher(); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(app.GetAppId (), 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container1 = new MockContainer(appAttemptId, dispatcher, conf, app.GetUser(), app.GetAppId (), 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container2 = new MockContainer(appAttemptId, dispatcher, conf, app.GetUser(), app.GetAppId (), 2); nmContext.GetContainers()[container1.GetContainerId()] = container1; nmContext.GetContainers()[container2.GetContainerId()] = container2; app.GetContainers()[container1.GetContainerId()] = container1; app.GetContainers()[container2.GetContainerId()] = container2; Dictionary <string, string> hash = new Dictionary <string, string>(); hash[container1.GetContainerId().ToString()] = container1.GetContainerId().ToString (); hash[container2.GetContainerId().ToString()] = container2.GetContainerId().ToString (); return(hash); }
internal WrappedApplication(TestApplication _enclosing, int id, long timestamp, string user, int numContainers) { this._enclosing = _enclosing; Configuration conf = new Configuration(); this.dispatcher = new DrainDispatcher(); this.containerTokenIdentifierMap = new Dictionary <ContainerId, ContainerTokenIdentifier >(); this.dispatcher.Init(conf); this.localizerBus = Org.Mockito.Mockito.Mock <EventHandler>(); this.launcherBus = Org.Mockito.Mockito.Mock <EventHandler>(); this.monitorBus = Org.Mockito.Mockito.Mock <EventHandler>(); this.auxBus = Org.Mockito.Mockito.Mock <EventHandler>(); this.containerBus = Org.Mockito.Mockito.Mock <EventHandler>(); this.logAggregationBus = Org.Mockito.Mockito.Mock <EventHandler>(); this.dispatcher.Register(typeof(LocalizationEventType), this.localizerBus); this.dispatcher.Register(typeof(ContainersLauncherEventType), this.launcherBus); this.dispatcher.Register(typeof(ContainersMonitorEventType), this.monitorBus); this.dispatcher.Register(typeof(AuxServicesEventType), this.auxBus); this.dispatcher.Register(typeof(ContainerEventType), this.containerBus); this.dispatcher.Register(typeof(LogHandlerEventType), this.logAggregationBus); this.nmTokenSecretMgr = Org.Mockito.Mockito.Mock <NMTokenSecretManagerInNM>(); this.context = Org.Mockito.Mockito.Mock <Context>(); Org.Mockito.Mockito.When(this.context.GetContainerTokenSecretManager()).ThenReturn (new NMContainerTokenSecretManager(conf)); Org.Mockito.Mockito.When(this.context.GetApplicationACLsManager()).ThenReturn(new ApplicationACLsManager(conf)); Org.Mockito.Mockito.When(this.context.GetNMTokenSecretManager()).ThenReturn(this. nmTokenSecretMgr); // Setting master key MasterKey masterKey = new MasterKeyPBImpl(); masterKey.SetKeyId(123); masterKey.SetBytes(ByteBuffer.Wrap(new byte[] { (123) })); this.context.GetContainerTokenSecretManager().SetMasterKey(masterKey); this.user = user; this.appId = BuilderUtils.NewApplicationId(timestamp, id); this.app = new ApplicationImpl(this.dispatcher, this.user, this.appId, null, this .context); this.containers = new AList <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(); for (int i = 0; i < numContainers; i++) { Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = this._enclosing.CreateMockedContainer(this.appId, i); this.containers.AddItem(container); long currentTime = Runtime.CurrentTimeMillis(); ContainerTokenIdentifier identifier = new ContainerTokenIdentifier(container.GetContainerId (), string.Empty, string.Empty, null, currentTime + 2000, masterKey.GetKeyId(), currentTime, Priority.NewInstance(0), 0); this.containerTokenIdentifierMap[identifier.GetContainerID()] = identifier; this.context.GetContainerTokenSecretManager().StartContainerSuccessful(identifier ); NUnit.Framework.Assert.IsFalse(this.context.GetContainerTokenSecretManager().IsValidStartContainerRequest (identifier)); } this.dispatcher.Start(); }
public RecoveredContainerLaunch(Context context, Configuration configuration, Dispatcher dispatcher, ContainerExecutor exec, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container, LocalDirsHandlerService dirsHandler, ContainerManagerImpl containerManager ) : base(context, configuration, dispatcher, exec, app, container, dirsHandler, containerManager ) { this.shouldLaunchContainer.Set(true); }
public virtual void Handle(ContainersLauncherEvent @event) { // TODO: ContainersLauncher launches containers one by one!! Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = @event.GetContainer(); ContainerId containerId = container.GetContainerId(); switch (@event.GetType()) { case ContainersLauncherEventType.LaunchContainer: { Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = context.GetApplications()[containerId.GetApplicationAttemptId().GetApplicationId ()]; ContainerLaunch launch = new ContainerLaunch(context, GetConfig(), dispatcher, exec , app, @event.GetContainer(), dirsHandler, containerManager); containerLauncher.Submit(launch); running[containerId] = launch; break; } case ContainersLauncherEventType.RecoverContainer: { app = context.GetApplications()[containerId.GetApplicationAttemptId().GetApplicationId ()]; launch = new RecoveredContainerLaunch(context, GetConfig(), dispatcher, exec, app , @event.GetContainer(), dirsHandler, containerManager); containerLauncher.Submit(launch); running[containerId] = launch; break; } case ContainersLauncherEventType.CleanupContainer: { ContainerLaunch launcher = Sharpen.Collections.Remove(running, containerId); if (launcher == null) { // Container not launched. So nothing needs to be done. return; } // Cleanup a container whether it is running/killed/completed, so that // no sub-processes are alive. try { launcher.CleanupContainer(); } catch (IOException) { Log.Warn("Got exception while cleaning container " + containerId + ". Ignoring."); } break; } } }
GetApplicationForContainer(ContainerId containerId, Context context) { ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId (); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application application = context.GetApplications()[applicationId]; if (application == null) { throw new NotFoundException("Unknown container. Container either has not started or " + "has already completed or " + "doesn't belong to this node at all."); } return(application); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual void TestContainerLogFile() { FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir" ).GetAbsoluteFile(); string logdirwithFile = absLogDir.ToURI().ToString(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile); conf.SetFloat(YarnConfiguration.NmMaxPerDiskUtilizationPercentage, 0.0f); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); dirsHandler.Init(conf); NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler , new ApplicationACLsManager(conf), new NMNullStateStoreService()); // Add an application and the corresponding containers string user = "******"; long clusterTimeStamp = 1234; ApplicationId appId = BuilderUtils.NewApplicationId(clusterTimeStamp, 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user); Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1); nmContext.GetApplications()[appId] = app; MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, appId, 1); container.SetState(ContainerState.Running); nmContext.GetContainers()[containerId] = container; FilePath containerLogDir = new FilePath(absLogDir, ContainerLaunch.GetRelativeContainerLogDir (appId.ToString(), containerId.ToString())); containerLogDir.Mkdirs(); string fileName = "fileName"; FilePath containerLogFile = new FilePath(containerLogDir, fileName); containerLogFile.CreateNewFile(); FilePath file = ContainerLogsUtils.GetContainerLogFile(containerId, fileName, user , nmContext); NUnit.Framework.Assert.AreEqual(containerLogFile.ToURI().ToString(), file.ToURI() .ToString()); FileUtil.FullyDelete(absLogDir); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> private static void CheckAccess(string remoteUser, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application application, Context context) { UserGroupInformation callerUGI = null; if (remoteUser != null) { callerUGI = UserGroupInformation.CreateRemoteUser(remoteUser); } if (callerUGI != null && !context.GetApplicationACLsManager().CheckAccess(callerUGI , ApplicationAccessType.ViewApp, application.GetUser(), application.GetAppId())) { throw new YarnException("User [" + remoteUser + "] is not authorized to view the logs for application " + application.GetAppId()); } }
public AppInfo(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app) { // JAXB needs this this.id = ConverterUtils.ToString(app.GetAppId()); this.state = app.GetApplicationState().ToString(); this.user = app.GetUser(); this.containerids = new AList <string>(); IDictionary <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container > appContainers = app.GetContainers(); foreach (ContainerId containerId in appContainers.Keys) { string containerIdStr = ConverterUtils.ToString(containerId); containerids.AddItem(containerIdStr); } }
public virtual void TestLogFileWithDriveLetter() { ContainerImpl container = Org.Mockito.Mockito.Mock <ContainerImpl>(); ApplicationIdPBImpl appId = Org.Mockito.Mockito.Mock <ApplicationIdPBImpl>(); Org.Mockito.Mockito.When(appId.ToString()).ThenReturn("appId"); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId); ApplicationAttemptIdPBImpl appAttemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptIdPBImpl >(); Org.Mockito.Mockito.When(appAttemptId.GetApplicationId()).ThenReturn(appId); ConcurrentMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application > applications = new ConcurrentHashMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); applications[appId] = app; ContainerId containerId = Org.Mockito.Mockito.Mock <ContainerIdPBImpl>(); Org.Mockito.Mockito.When(containerId.ToString()).ThenReturn("containerId"); Org.Mockito.Mockito.When(containerId.GetApplicationAttemptId()).ThenReturn(appAttemptId ); ConcurrentMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container > containers = new ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(); containers[containerId] = container; LocalDirsHandlerService localDirs = Org.Mockito.Mockito.Mock <LocalDirsHandlerService >(); Org.Mockito.Mockito.When(localDirs.GetLogPathToRead("appId" + Path.Separator + "containerId" + Path.Separator + "fileName")).ThenReturn(new Path("F:/nmlogs/appId/containerId/fileName" )); NodeManager.NMContext context = Org.Mockito.Mockito.Mock <NodeManager.NMContext>(); Org.Mockito.Mockito.When(context.GetLocalDirsHandler()).ThenReturn(localDirs); Org.Mockito.Mockito.When(context.GetApplications()).ThenReturn(applications); Org.Mockito.Mockito.When(context.GetContainers()).ThenReturn(containers); FilePath logFile = ContainerLogsUtils.GetContainerLogFile(containerId, "fileName" , null, context); NUnit.Framework.Assert.IsTrue("logFile lost drive letter " + logFile, logFile.ToString ().IndexOf("F:" + FilePath.separator + "nmlogs") > -1); }
public ContainerLaunch(Context context, Configuration configuration, Dispatcher dispatcher , ContainerExecutor exec, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container, LocalDirsHandlerService dirsHandler, ContainerManagerImpl containerManager ) { this.context = context; this.conf = configuration; this.app = app; this.exec = exec; this.container = container; this.dispatcher = dispatcher; this.dirsHandler = dirsHandler; this.containerManager = containerManager; this.sleepDelayBeforeSigKill = conf.GetLong(YarnConfiguration.NmSleepDelayBeforeSigkillMs , YarnConfiguration.DefaultNmSleepDelayBeforeSigkillMs); this.maxKillWaitTime = conf.GetLong(YarnConfiguration.NmProcessKillWaitMs, YarnConfiguration .DefaultNmProcessKillWaitMs); }
protected override void Render(HtmlBlock.Block html) { ApplicationId applicationID = ConverterUtils.ToApplicationId(this.recordFactory, $(ApplicationId)); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = this.nmContext.GetApplications()[applicationID]; AppInfo info = new AppInfo(app); Info("Application's information").("ApplicationId", info.GetId()).("ApplicationState" , info.GetState()).("User", info.GetUser()); Hamlet.TABLE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> containersListBody = html .(typeof(InfoBlock)).Table("#containers"); foreach (string containerIdStr in info.GetContainers()) { containersListBody.Tr().Td().A(Url("container", containerIdStr), containerIdStr). ().(); } containersListBody.(); }
/// <summary> /// Finds the local directories that logs for the given container are stored /// on. /// </summary> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public static IList <FilePath> GetContainerLogDirs(ContainerId containerId, string remoteUser, Context context) { Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = context.GetContainers()[containerId]; Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application application = GetApplicationForContainer(containerId, context); CheckAccess(remoteUser, application, context); // It is not required to have null check for container ( container == null ) // and throw back exception.Because when container is completed, NodeManager // remove container information from its NMContext.Configuring log // aggregation to false, container log view request is forwarded to NM. NM // does not have completed container information,but still NM serve request for // reading container logs. if (container != null) { CheckState(container.GetContainerState()); } return(GetContainerLogDirs(containerId, context.GetLocalDirsHandler())); }
public virtual void Logs() { string containerIdStr = $(ContainerId); ContainerId containerId = null; try { containerId = ConverterUtils.ToContainerId(containerIdStr); } catch (ArgumentException) { Render(typeof(ContainerLogsPage)); return; } ApplicationId appId = containerId.GetApplicationAttemptId().GetApplicationId(); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = nmContext.GetApplications()[appId]; if (app == null && nmConf.GetBoolean(YarnConfiguration.LogAggregationEnabled, YarnConfiguration .DefaultLogAggregationEnabled)) { string logServerUrl = nmConf.Get(YarnConfiguration.YarnLogServerUrl); string redirectUrl = null; if (logServerUrl == null || logServerUrl.IsEmpty()) { redirectUrl = "false"; } else { redirectUrl = Url(logServerUrl, nmContext.GetNodeId().ToString(), containerIdStr, containerIdStr, $(AppOwner)); } Set(ContainerLogsPage.RedirectUrl, redirectUrl); } Render(typeof(ContainerLogsPage)); }
public ApplicationLocalizationEvent(LocalizationEventType type, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app) : base(type) { this.app = app; }
/// <exception cref="System.IO.IOException"/> public virtual void TestContainerLogPageAccess() { // SecureIOUtils require Native IO to be enabled. This test will run // only if it is enabled. Assume.AssumeTrue(NativeIO.IsAvailable()); string user = "******" + Runtime.CurrentTimeMillis(); FilePath absLogDir = null; FilePath appDir = null; FilePath containerDir = null; FilePath syslog = null; try { // target log directory absLogDir = new FilePath("target", typeof(TestContainerLogsPage).Name + "LogDir") .GetAbsoluteFile(); absLogDir.Mkdir(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLogDirs, absLogDir.ToURI().ToString()); conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); healthChecker.Init(conf); LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler(); // Add an application and the corresponding containers RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf); long clusterTimeStamp = 1234; ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp , 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId); // Making sure that application returns a random user. This is required // for SecureIOUtils' file owner check. Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId , 0); // Testing secure read access for log files // Creating application and container directory and syslog file. appDir = new FilePath(absLogDir, appId.ToString()); appDir.Mkdir(); containerDir = new FilePath(appDir, container1.ToString()); containerDir.Mkdir(); syslog = new FilePath(containerDir, "syslog"); syslog.CreateNewFile(); BufferedOutputStream @out = new BufferedOutputStream(new FileOutputStream(syslog) ); @out.Write(Sharpen.Runtime.GetBytesForString("Log file Content")); @out.Close(); Context context = Org.Mockito.Mockito.Mock <Context>(); ConcurrentMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application > appMap = new ConcurrentHashMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); appMap[appId] = app; Org.Mockito.Mockito.When(context.GetApplications()).ThenReturn(appMap); ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container > containers = new ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(); Org.Mockito.Mockito.When(context.GetContainers()).ThenReturn(containers); Org.Mockito.Mockito.When(context.GetLocalDirsHandler()).ThenReturn(dirsHandler); MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, appId, 1); container.SetState(ContainerState.Running); context.GetContainers()[container1] = container; ContainerLogsPage.ContainersLogsBlock cLogsBlock = new ContainerLogsPage.ContainersLogsBlock (context); IDictionary <string, string> @params = new Dictionary <string, string>(); @params[YarnWebParams.ContainerId] = container1.ToString(); @params[YarnWebParams.ContainerLogType] = "syslog"; Injector injector = WebAppTests.TestPage <ContainerLogsPage.ContainersLogsBlock>(typeof( ContainerLogsPage), cLogsBlock, @params, (Module[])null); PrintWriter spyPw = WebAppTests.GetPrintWriter(injector); Org.Mockito.Mockito.Verify(spyPw).Write("Exception reading log file. Application submitted by '" + user + "' doesn't own requested log file : syslog"); } finally { if (syslog != null) { syslog.Delete(); } if (containerDir != null) { containerDir.Delete(); } if (appDir != null) { appDir.Delete(); } if (absLogDir != null) { absLogDir.Delete(); } } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual void TestContainerLogDirs() { FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir" ).GetAbsoluteFile(); string logdirwithFile = absLogDir.ToURI().ToString(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); healthChecker.Init(conf); LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler(); NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler , new ApplicationACLsManager(conf), new NMNullStateStoreService()); // Add an application and the corresponding containers RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf); string user = "******"; long clusterTimeStamp = 1234; ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp , 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user); Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId , 0); nmContext.GetApplications()[appId] = app; MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, appId, 1); container.SetState(ContainerState.Running); nmContext.GetContainers()[container1] = container; IList <FilePath> files = null; files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext); NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:"))); // After container is completed, it is removed from nmContext Sharpen.Collections.Remove(nmContext.GetContainers(), container1); NUnit.Framework.Assert.IsNull(nmContext.GetContainers()[container1]); files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext); NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:"))); // Create a new context to check if correct container log dirs are fetched // on full disk. LocalDirsHandlerService dirsHandlerForFullDisk = Org.Mockito.Mockito.Spy(dirsHandler ); // good log dirs are empty and nm log dir is in the full log dir list. Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirs()).ThenReturn(new AList <string>()); Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirsForRead()).ThenReturn(Arrays .AsList(new string[] { absLogDir.GetAbsolutePath() })); nmContext = new NodeManager.NMContext(null, null, dirsHandlerForFullDisk, new ApplicationACLsManager (conf), new NMNullStateStoreService()); nmContext.GetApplications()[appId] = app; container.SetState(ContainerState.Running); nmContext.GetContainers()[container1] = container; IList <FilePath> dirs = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext ); FilePath containerLogDir = new FilePath(absLogDir, appId + "/" + container1); NUnit.Framework.Assert.IsTrue(dirs.Contains(containerLogDir)); }
public ContainerLocalizationImpl(Dispatcher dispatcher, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app, LocalizationProtocol localization) { }