public virtual void TestLogDeletion() { FilePath[] localLogDirs = GetLocalLogDirFiles(this.GetType().FullName, 2); string localLogDirsString = localLogDirs[0].GetAbsolutePath() + "," + localLogDirs [1].GetAbsolutePath(); conf.Set(YarnConfiguration.NmLogDirs, localLogDirsString); conf.SetBoolean(YarnConfiguration.LogAggregationEnabled, false); conf.SetLong(YarnConfiguration.NmLogRetainSeconds, 0l); dirsHandler.Init(conf); NonAggregatingLogHandler rawLogHandler = new NonAggregatingLogHandler(dispatcher, mockDelService, dirsHandler, new NMNullStateStoreService()); NonAggregatingLogHandler logHandler = Org.Mockito.Mockito.Spy(rawLogHandler); AbstractFileSystem spylfs = Org.Mockito.Mockito.Spy(FileContext.GetLocalFSFileContext ().GetDefaultFileSystem()); FileContext lfs = FileContext.GetFileContext(spylfs, conf); Org.Mockito.Mockito.DoReturn(lfs).When(logHandler).GetLocalFileContext(Matchers.IsA <Configuration>()); FsPermission defaultPermission = FsPermission.GetDirDefault().ApplyUMask(lfs.GetUMask ()); FileStatus fs = new FileStatus(0, true, 1, 0, Runtime.CurrentTimeMillis(), 0, defaultPermission , string.Empty, string.Empty, new Path(localLogDirs[0].GetAbsolutePath())); Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.IsA <Path>()); logHandler.Init(conf); logHandler.Start(); logHandler.Handle(new LogHandlerAppStartedEvent(appId, user, null, ContainerLogsRetentionPolicy .AllContainers, null)); logHandler.Handle(new LogHandlerContainerFinishedEvent(container11, 0)); logHandler.Handle(new LogHandlerAppFinishedEvent(appId)); Path[] localAppLogDirs = new Path[2]; localAppLogDirs[0] = new Path(localLogDirs[0].GetAbsolutePath(), appId.ToString() ); localAppLogDirs[1] = new Path(localLogDirs[1].GetAbsolutePath(), appId.ToString() ); TestDeletionServiceCall(mockDelService, user, 5000, localAppLogDirs); logHandler.Close(); for (int i = 0; i < localLogDirs.Length; i++) { FileUtils.DeleteDirectory(localLogDirs[i]); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual void TestContainerLogFile() { FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir" ).GetAbsoluteFile(); string logdirwithFile = absLogDir.ToURI().ToString(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile); conf.SetFloat(YarnConfiguration.NmMaxPerDiskUtilizationPercentage, 0.0f); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); dirsHandler.Init(conf); NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler , new ApplicationACLsManager(conf), new NMNullStateStoreService()); // Add an application and the corresponding containers string user = "******"; long clusterTimeStamp = 1234; ApplicationId appId = BuilderUtils.NewApplicationId(clusterTimeStamp, 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user); Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1); nmContext.GetApplications()[appId] = app; MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, appId, 1); container.SetState(ContainerState.Running); nmContext.GetContainers()[containerId] = container; FilePath containerLogDir = new FilePath(absLogDir, ContainerLaunch.GetRelativeContainerLogDir (appId.ToString(), containerId.ToString())); containerLogDir.Mkdirs(); string fileName = "fileName"; FilePath containerLogFile = new FilePath(containerLogDir, fileName); containerLogFile.CreateNewFile(); FilePath file = ContainerLogsUtils.GetContainerLogFile(containerId, fileName, user , nmContext); NUnit.Framework.Assert.AreEqual(containerLogFile.ToURI().ToString(), file.ToURI() .ToString()); FileUtil.FullyDelete(absLogDir); }
public virtual void TestDirFailuresOnStartup() { Configuration conf = new YarnConfiguration(); string localDir1 = new FilePath(testDir, "localDir1").GetPath(); string localDir2 = new FilePath(testDir, "localDir2").GetPath(); string logDir1 = new FilePath(testDir, "logDir1").GetPath(); string logDir2 = new FilePath(testDir, "logDir2").GetPath(); conf.Set(YarnConfiguration.NmLocalDirs, localDir1 + "," + localDir2); conf.Set(YarnConfiguration.NmLogDirs, logDir1 + "," + logDir2); PrepareDirToFail(localDir1); PrepareDirToFail(logDir2); LocalDirsHandlerService dirSvc = new LocalDirsHandlerService(); dirSvc.Init(conf); IList <string> localDirs = dirSvc.GetLocalDirs(); NUnit.Framework.Assert.AreEqual(1, localDirs.Count); NUnit.Framework.Assert.AreEqual(new Path(localDir2).ToString(), localDirs[0]); IList <string> logDirs = dirSvc.GetLogDirs(); NUnit.Framework.Assert.AreEqual(1, logDirs.Count); NUnit.Framework.Assert.AreEqual(new Path(logDir1).ToString(), logDirs[0]); }