public virtual void TestContainerLogs() { WebResource r = Resource(); ContainerId containerId = BuilderUtils.NewContainerId(0, 0, 0, 0); string containerIdStr = BuilderUtils.NewContainerId(0, 0, 0, 0).ToString(); ApplicationAttemptId appAttemptId = containerId.GetApplicationAttemptId(); ApplicationId appId = appAttemptId.GetApplicationId(); string appIdStr = appId.ToString(); string filename = "logfile1"; string logMessage = "log message\n"; nmContext.GetApplications()[appId] = new ApplicationImpl(null, "user", appId, null , nmContext); MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), new Configuration(), "user", appId, 1); container.SetState(ContainerState.Running); nmContext.GetContainers()[containerId] = container; // write out log file Path path = dirsHandler.GetLogPathForWrite(ContainerLaunch.GetRelativeContainerLogDir (appIdStr, containerIdStr) + "/" + filename, false); FilePath logFile = new FilePath(path.ToUri().GetPath()); logFile.DeleteOnExit(); NUnit.Framework.Assert.IsTrue("Failed to create log dir", logFile.GetParentFile() .Mkdirs()); PrintWriter pw = new PrintWriter(logFile); pw.Write(logMessage); pw.Close(); // ask for it ClientResponse response = r.Path("ws").Path("v1").Path("node").Path("containerlogs" ).Path(containerIdStr).Path(filename).Accept(MediaType.TextPlain).Get <ClientResponse >(); string responseText = response.GetEntity <string>(); NUnit.Framework.Assert.AreEqual(logMessage, responseText); // ask for file that doesn't exist response = r.Path("ws").Path("v1").Path("node").Path("containerlogs").Path(containerIdStr ).Path("uhhh").Accept(MediaType.TextPlain).Get <ClientResponse>(); NUnit.Framework.Assert.AreEqual(ClientResponse.Status.NotFound.GetStatusCode(), response .GetStatus()); responseText = response.GetEntity <string>(); NUnit.Framework.Assert.IsTrue(responseText.Contains("Cannot find this log on the local disk." )); // After container is completed, it is removed from nmContext Sharpen.Collections.Remove(nmContext.GetContainers(), containerId); NUnit.Framework.Assert.IsNull(nmContext.GetContainers()[containerId]); response = r.Path("ws").Path("v1").Path("node").Path("containerlogs").Path(containerIdStr ).Path(filename).Accept(MediaType.TextPlain).Get <ClientResponse>(); responseText = response.GetEntity <string>(); NUnit.Framework.Assert.AreEqual(logMessage, responseText); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual void TestContainerLogFile() { FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir" ).GetAbsoluteFile(); string logdirwithFile = absLogDir.ToURI().ToString(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile); conf.SetFloat(YarnConfiguration.NmMaxPerDiskUtilizationPercentage, 0.0f); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); dirsHandler.Init(conf); NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler , new ApplicationACLsManager(conf), new NMNullStateStoreService()); // Add an application and the corresponding containers string user = "******"; long clusterTimeStamp = 1234; ApplicationId appId = BuilderUtils.NewApplicationId(clusterTimeStamp, 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user); Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1); nmContext.GetApplications()[appId] = app; MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, appId, 1); container.SetState(ContainerState.Running); nmContext.GetContainers()[containerId] = container; FilePath containerLogDir = new FilePath(absLogDir, ContainerLaunch.GetRelativeContainerLogDir (appId.ToString(), containerId.ToString())); containerLogDir.Mkdirs(); string fileName = "fileName"; FilePath containerLogFile = new FilePath(containerLogDir, fileName); containerLogFile.CreateNewFile(); FilePath file = ContainerLogsUtils.GetContainerLogFile(containerId, fileName, user , nmContext); NUnit.Framework.Assert.AreEqual(containerLogFile.ToURI().ToString(), file.ToURI() .ToString()); FileUtil.FullyDelete(absLogDir); }
/// <exception cref="System.IO.IOException"/> private Dictionary <string, string> AddAppContainers(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app) { Dispatcher dispatcher = new AsyncDispatcher(); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(app.GetAppId (), 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container1 = new MockContainer(appAttemptId, dispatcher, conf, app.GetUser(), app.GetAppId (), 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container2 = new MockContainer(appAttemptId, dispatcher, conf, app.GetUser(), app.GetAppId (), 2); nmContext.GetContainers()[container1.GetContainerId()] = container1; nmContext.GetContainers()[container2.GetContainerId()] = container2; app.GetContainers()[container1.GetContainerId()] = container1; app.GetContainers()[container2.GetContainerId()] = container2; Dictionary <string, string> hash = new Dictionary <string, string>(); hash[container1.GetContainerId().ToString()] = container1.GetContainerId().ToString (); hash[container2.GetContainerId().ToString()] = container2.GetContainerId().ToString (); return(hash); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual void TestContainerLogDirs() { FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir" ).GetAbsoluteFile(); string logdirwithFile = absLogDir.ToURI().ToString(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); healthChecker.Init(conf); LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler(); NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler , new ApplicationACLsManager(conf), new NMNullStateStoreService()); // Add an application and the corresponding containers RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf); string user = "******"; long clusterTimeStamp = 1234; ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp , 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user); Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId , 0); nmContext.GetApplications()[appId] = app; MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, appId, 1); container.SetState(ContainerState.Running); nmContext.GetContainers()[container1] = container; IList <FilePath> files = null; files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext); NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:"))); // After container is completed, it is removed from nmContext Sharpen.Collections.Remove(nmContext.GetContainers(), container1); NUnit.Framework.Assert.IsNull(nmContext.GetContainers()[container1]); files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext); NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:"))); // Create a new context to check if correct container log dirs are fetched // on full disk. LocalDirsHandlerService dirsHandlerForFullDisk = Org.Mockito.Mockito.Spy(dirsHandler ); // good log dirs are empty and nm log dir is in the full log dir list. Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirs()).ThenReturn(new AList <string>()); Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirsForRead()).ThenReturn(Arrays .AsList(new string[] { absLogDir.GetAbsolutePath() })); nmContext = new NodeManager.NMContext(null, null, dirsHandlerForFullDisk, new ApplicationACLsManager (conf), new NMNullStateStoreService()); nmContext.GetApplications()[appId] = app; container.SetState(ContainerState.Running); nmContext.GetContainers()[container1] = container; IList <FilePath> dirs = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext ); FilePath containerLogDir = new FilePath(absLogDir, appId + "/" + container1); NUnit.Framework.Assert.IsTrue(dirs.Contains(containerLogDir)); }
/// <exception cref="System.IO.IOException"/> public virtual void TestContainerLogPageAccess() { // SecureIOUtils require Native IO to be enabled. This test will run // only if it is enabled. Assume.AssumeTrue(NativeIO.IsAvailable()); string user = "******" + Runtime.CurrentTimeMillis(); FilePath absLogDir = null; FilePath appDir = null; FilePath containerDir = null; FilePath syslog = null; try { // target log directory absLogDir = new FilePath("target", typeof(TestContainerLogsPage).Name + "LogDir") .GetAbsoluteFile(); absLogDir.Mkdir(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLogDirs, absLogDir.ToURI().ToString()); conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); healthChecker.Init(conf); LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler(); // Add an application and the corresponding containers RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf); long clusterTimeStamp = 1234; ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp , 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId); // Making sure that application returns a random user. This is required // for SecureIOUtils' file owner check. Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId , 0); // Testing secure read access for log files // Creating application and container directory and syslog file. appDir = new FilePath(absLogDir, appId.ToString()); appDir.Mkdir(); containerDir = new FilePath(appDir, container1.ToString()); containerDir.Mkdir(); syslog = new FilePath(containerDir, "syslog"); syslog.CreateNewFile(); BufferedOutputStream @out = new BufferedOutputStream(new FileOutputStream(syslog) ); @out.Write(Sharpen.Runtime.GetBytesForString("Log file Content")); @out.Close(); Context context = Org.Mockito.Mockito.Mock <Context>(); ConcurrentMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application > appMap = new ConcurrentHashMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); appMap[appId] = app; Org.Mockito.Mockito.When(context.GetApplications()).ThenReturn(appMap); ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container > containers = new ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(); Org.Mockito.Mockito.When(context.GetContainers()).ThenReturn(containers); Org.Mockito.Mockito.When(context.GetLocalDirsHandler()).ThenReturn(dirsHandler); MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, appId, 1); container.SetState(ContainerState.Running); context.GetContainers()[container1] = container; ContainerLogsPage.ContainersLogsBlock cLogsBlock = new ContainerLogsPage.ContainersLogsBlock (context); IDictionary <string, string> @params = new Dictionary <string, string>(); @params[YarnWebParams.ContainerId] = container1.ToString(); @params[YarnWebParams.ContainerLogType] = "syslog"; Injector injector = WebAppTests.TestPage <ContainerLogsPage.ContainersLogsBlock>(typeof( ContainerLogsPage), cLogsBlock, @params, (Module[])null); PrintWriter spyPw = WebAppTests.GetPrintWriter(injector); Org.Mockito.Mockito.Verify(spyPw).Write("Exception reading log file. Application submitted by '" + user + "' doesn't own requested log file : syslog"); } finally { if (syslog != null) { syslog.Delete(); } if (containerDir != null) { containerDir.Delete(); } if (appDir != null) { appDir.Delete(); } if (absLogDir != null) { absLogDir.Delete(); } } }