public virtual void TestLogDirWithDriveLetter() { //To verify that logs paths which include drive letters (Windows) //do not lose their drive letter specification LocalDirsHandlerService localDirs = Org.Mockito.Mockito.Mock <LocalDirsHandlerService >(); IList <string> logDirs = new AList <string>(); logDirs.AddItem("F:/nmlogs"); Org.Mockito.Mockito.When(localDirs.GetLogDirsForRead()).ThenReturn(logDirs); ApplicationIdPBImpl appId = Org.Mockito.Mockito.Mock <ApplicationIdPBImpl>(); Org.Mockito.Mockito.When(appId.ToString()).ThenReturn("app_id_1"); ApplicationAttemptIdPBImpl appAttemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptIdPBImpl >(); Org.Mockito.Mockito.When(appAttemptId.GetApplicationId()).ThenReturn(appId); ContainerId containerId = Org.Mockito.Mockito.Mock <ContainerIdPBImpl>(); Org.Mockito.Mockito.When(containerId.GetApplicationAttemptId()).ThenReturn(appAttemptId ); IList <FilePath> logDirFiles = ContainerLogsUtils.GetContainerLogDirs(containerId, localDirs); NUnit.Framework.Assert.IsTrue("logDir lost drive letter " + logDirFiles[0], logDirFiles [0].ToString().IndexOf("F:" + FilePath.separator + "nmlogs") > -1); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual void TestContainerLogFile() { FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir" ).GetAbsoluteFile(); string logdirwithFile = absLogDir.ToURI().ToString(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile); conf.SetFloat(YarnConfiguration.NmMaxPerDiskUtilizationPercentage, 0.0f); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); dirsHandler.Init(conf); NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler , new ApplicationACLsManager(conf), new NMNullStateStoreService()); // Add an application and the corresponding containers string user = "******"; long clusterTimeStamp = 1234; ApplicationId appId = BuilderUtils.NewApplicationId(clusterTimeStamp, 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user); Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1); nmContext.GetApplications()[appId] = app; MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, appId, 1); container.SetState(ContainerState.Running); nmContext.GetContainers()[containerId] = container; FilePath containerLogDir = new FilePath(absLogDir, ContainerLaunch.GetRelativeContainerLogDir (appId.ToString(), containerId.ToString())); containerLogDir.Mkdirs(); string fileName = "fileName"; FilePath containerLogFile = new FilePath(containerLogDir, fileName); containerLogFile.CreateNewFile(); FilePath file = ContainerLogsUtils.GetContainerLogFile(containerId, fileName, user , nmContext); NUnit.Framework.Assert.AreEqual(containerLogFile.ToURI().ToString(), file.ToURI() .ToString()); FileUtil.FullyDelete(absLogDir); }
// TODO: Pull logs and test contents. // Thread.sleep(1000000); /// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> private void WriteContainerLogs(Context nmContext, ContainerId containerId, LocalDirsHandlerService dirsHandler) { // ContainerLogDir should be created FilePath containerLogDir = ContainerLogsUtils.GetContainerLogDirs(containerId, dirsHandler )[0]; containerLogDir.Mkdirs(); foreach (string fileType in new string[] { "stdout", "stderr", "syslog" }) { TextWriter writer = new FileWriter(new FilePath(containerLogDir, fileType)); writer.Write(ConverterUtils.ToString(containerId) + "\n Hello " + fileType + "!"); writer.Close(); } }
public virtual void TestLogFileWithDriveLetter() { ContainerImpl container = Org.Mockito.Mockito.Mock <ContainerImpl>(); ApplicationIdPBImpl appId = Org.Mockito.Mockito.Mock <ApplicationIdPBImpl>(); Org.Mockito.Mockito.When(appId.ToString()).ThenReturn("appId"); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId); ApplicationAttemptIdPBImpl appAttemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptIdPBImpl >(); Org.Mockito.Mockito.When(appAttemptId.GetApplicationId()).ThenReturn(appId); ConcurrentMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application > applications = new ConcurrentHashMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); applications[appId] = app; ContainerId containerId = Org.Mockito.Mockito.Mock <ContainerIdPBImpl>(); Org.Mockito.Mockito.When(containerId.ToString()).ThenReturn("containerId"); Org.Mockito.Mockito.When(containerId.GetApplicationAttemptId()).ThenReturn(appAttemptId ); ConcurrentMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container > containers = new ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(); containers[containerId] = container; LocalDirsHandlerService localDirs = Org.Mockito.Mockito.Mock <LocalDirsHandlerService >(); Org.Mockito.Mockito.When(localDirs.GetLogPathToRead("appId" + Path.Separator + "containerId" + Path.Separator + "fileName")).ThenReturn(new Path("F:/nmlogs/appId/containerId/fileName" )); NodeManager.NMContext context = Org.Mockito.Mockito.Mock <NodeManager.NMContext>(); Org.Mockito.Mockito.When(context.GetLocalDirsHandler()).ThenReturn(localDirs); Org.Mockito.Mockito.When(context.GetApplications()).ThenReturn(applications); Org.Mockito.Mockito.When(context.GetContainers()).ThenReturn(containers); FilePath logFile = ContainerLogsUtils.GetContainerLogFile(containerId, "fileName" , null, context); NUnit.Framework.Assert.IsTrue("logFile lost drive letter " + logFile, logFile.ToString ().IndexOf("F:" + FilePath.separator + "nmlogs") > -1); }
public virtual Response GetLogs(string containerIdStr, string filename) { ContainerId containerId; try { containerId = ConverterUtils.ToContainerId(containerIdStr); } catch (ArgumentException) { return(Response.Status(Response.Status.BadRequest).Build()); } FilePath logFile = null; try { logFile = ContainerLogsUtils.GetContainerLogFile(containerId, filename, request.GetRemoteUser (), nmContext); } catch (NotFoundException ex) { return(Response.Status(Response.Status.NotFound).Entity(ex.Message).Build()); } catch (YarnException ex) { return(Response.ServerError().Entity(ex.Message).Build()); } try { FileInputStream fis = ContainerLogsUtils.OpenLogFileForRead(containerIdStr, logFile , nmContext); StreamingOutput stream = new _StreamingOutput_240(fis); return(Response.Ok(stream).Build()); } catch (IOException ex) { return(Response.ServerError().Entity(ex.Message).Build()); } }
protected override void Render(HtmlBlock.Block html) { string redirectUrl = $(RedirectUrl); if (redirectUrl != null && redirectUrl.Equals("false")) { html.H1("Failed while trying to construct the redirect url to the log" + " server. Log Server url may not be configured" ); } //Intentional fallthrough. ContainerId containerId; try { containerId = ConverterUtils.ToContainerId($(YarnWebParams.ContainerId)); } catch (ArgumentException) { html.H1("Invalid container ID: " + $(YarnWebParams.ContainerId)); return; } try { if ($(ContainerLogType).IsEmpty()) { IList <FilePath> logFiles = ContainerLogsUtils.GetContainerLogDirs(containerId, Request ().GetRemoteUser(), nmContext); PrintLogFileDirectory(html, logFiles); } else { FilePath logFile = ContainerLogsUtils.GetContainerLogFile(containerId, $(ContainerLogType ), Request().GetRemoteUser(), nmContext); PrintLogFile(html, logFile); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual void TestContainerLogDirs() { FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir" ).GetAbsoluteFile(); string logdirwithFile = absLogDir.ToURI().ToString(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); healthChecker.Init(conf); LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler(); NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler , new ApplicationACLsManager(conf), new NMNullStateStoreService()); // Add an application and the corresponding containers RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf); string user = "******"; long clusterTimeStamp = 1234; ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp , 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user); Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId , 0); nmContext.GetApplications()[appId] = app; MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, appId, 1); container.SetState(ContainerState.Running); nmContext.GetContainers()[container1] = container; IList <FilePath> files = null; files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext); NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:"))); // After container is completed, it is removed from nmContext Sharpen.Collections.Remove(nmContext.GetContainers(), container1); NUnit.Framework.Assert.IsNull(nmContext.GetContainers()[container1]); files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext); NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:"))); // Create a new context to check if correct container log dirs are fetched // on full disk. LocalDirsHandlerService dirsHandlerForFullDisk = Org.Mockito.Mockito.Spy(dirsHandler ); // good log dirs are empty and nm log dir is in the full log dir list. Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirs()).ThenReturn(new AList <string>()); Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirsForRead()).ThenReturn(Arrays .AsList(new string[] { absLogDir.GetAbsolutePath() })); nmContext = new NodeManager.NMContext(null, null, dirsHandlerForFullDisk, new ApplicationACLsManager (conf), new NMNullStateStoreService()); nmContext.GetApplications()[appId] = app; container.SetState(ContainerState.Running); nmContext.GetContainers()[container1] = container; IList <FilePath> dirs = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext ); FilePath containerLogDir = new FilePath(absLogDir, appId + "/" + container1); NUnit.Framework.Assert.IsTrue(dirs.Contains(containerLogDir)); }