public virtual void Setup() { localFS.Delete(new Path(localDir.GetAbsolutePath()), true); localFS.Delete(new Path(tmpDir.GetAbsolutePath()), true); localFS.Delete(new Path(localLogDir.GetAbsolutePath()), true); localFS.Delete(new Path(remoteLogDir.GetAbsolutePath()), true); localDir.Mkdir(); tmpDir.Mkdir(); localLogDir.Mkdir(); remoteLogDir.Mkdir(); Log.Info("Created localDir in " + localDir.GetAbsolutePath()); Log.Info("Created tmpDir in " + tmpDir.GetAbsolutePath()); string bindAddress = "0.0.0.0:12345"; conf.Set(YarnConfiguration.NmAddress, bindAddress); conf.Set(YarnConfiguration.NmLocalDirs, localDir.GetAbsolutePath()); conf.Set(YarnConfiguration.NmLogDirs, localLogDir.GetAbsolutePath()); conf.Set(YarnConfiguration.NmRemoteAppLogDir, remoteLogDir.GetAbsolutePath()); conf.SetLong(YarnConfiguration.NmLogRetainSeconds, 1); // Default delSrvc delSrvc = CreateDeletionService(); delSrvc.Init(conf); exec = CreateContainerExecutor(); nodeHealthChecker = new NodeHealthCheckerService(); nodeHealthChecker.Init(conf); dirsHandler = nodeHealthChecker.GetDiskHandler(); containerManager = CreateContainerManager(delSrvc); ((NodeManager.NMContext)context).SetContainerManager(containerManager); nodeStatusUpdater.Init(conf); containerManager.Init(conf); nodeStatusUpdater.Start(); }
public virtual void TestHandlingApplicationFinishedEvent() { DeletionService delService = new DeletionService(null); NonAggregatingLogHandler aggregatingLogHandler = new NonAggregatingLogHandler(new InlineDispatcher(), delService, dirsHandler, new NMNullStateStoreService()); dirsHandler.Init(conf); dirsHandler.Start(); delService.Init(conf); delService.Start(); aggregatingLogHandler.Init(conf); aggregatingLogHandler.Start(); // It should NOT throw RejectedExecutionException aggregatingLogHandler.Handle(new LogHandlerAppFinishedEvent(appId)); aggregatingLogHandler.Stop(); // It should NOT throw RejectedExecutionException after stopping // handler service. aggregatingLogHandler.Handle(new LogHandlerAppFinishedEvent(appId)); aggregatingLogHandler.Close(); }
public virtual void TestLocalFilesCleanup() { // Real del service delSrvc = new DeletionService(exec); delSrvc.Init(conf); containerManager = CreateContainerManager(delSrvc); containerManager.Init(conf); containerManager.Start(); // ////// Create the resources for the container FilePath dir = new FilePath(tmpDir, "dir"); dir.Mkdirs(); FilePath file = new FilePath(dir, "file"); PrintWriter fileWriter = new PrintWriter(file); fileWriter.Write("Hello World!"); fileWriter.Close(); // ////// Construct the Container-id ContainerId cId = CreateContainerId(0); ApplicationId appId = cId.GetApplicationAttemptId().GetApplicationId(); // ////// Construct the container-spec. ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); // containerLaunchContext.resources = // new HashMap<CharSequence, LocalResource>(); URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(FileContext.GetLocalFSFileContext ().MakeQualified(new Path(file.GetAbsolutePath()))); LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>(); rsrc_alpha.SetResource(resource_alpha); rsrc_alpha.SetSize(-1); rsrc_alpha.SetVisibility(LocalResourceVisibility.Application); rsrc_alpha.SetType(LocalResourceType.File); rsrc_alpha.SetTimestamp(file.LastModified()); string destinationFile = "dest_file"; IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); localResources[destinationFile] = rsrc_alpha; containerLaunchContext.SetLocalResources(localResources); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context .GetContainerTokenSecretManager())); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); containerManager.StartContainers(allRequests); BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState .Complete); BaseContainerManagerTest.WaitForApplicationState(containerManager, cId.GetApplicationAttemptId ().GetApplicationId(), ApplicationState.Running); // Now ascertain that the resources are localised correctly. string appIDStr = ConverterUtils.ToString(appId); string containerIDStr = ConverterUtils.ToString(cId); FilePath userCacheDir = new FilePath(localDir, ContainerLocalizer.Usercache); FilePath userDir = new FilePath(userCacheDir, user); FilePath appCache = new FilePath(userDir, ContainerLocalizer.Appcache); FilePath appDir = new FilePath(appCache, appIDStr); FilePath containerDir = new FilePath(appDir, containerIDStr); FilePath targetFile = new FilePath(containerDir, destinationFile); FilePath sysDir = new FilePath(localDir, ResourceLocalizationService.NmPrivateDir ); FilePath appSysDir = new FilePath(sysDir, appIDStr); FilePath containerSysDir = new FilePath(appSysDir, containerIDStr); // AppDir should still exist NUnit.Framework.Assert.IsTrue("AppDir " + appDir.GetAbsolutePath() + " doesn't exist!!" , appDir.Exists()); NUnit.Framework.Assert.IsTrue("AppSysDir " + appSysDir.GetAbsolutePath() + " doesn't exist!!" , appSysDir.Exists()); foreach (FilePath f in new FilePath[] { containerDir, containerSysDir }) { NUnit.Framework.Assert.IsFalse(f.GetAbsolutePath() + " exists!!", f.Exists()); } NUnit.Framework.Assert.IsFalse(targetFile.GetAbsolutePath() + " exists!!", targetFile .Exists()); // Simulate RM sending an AppFinish event. containerManager.Handle(new CMgrCompletedAppsEvent(Arrays.AsList(new ApplicationId [] { appId }), CMgrCompletedAppsEvent.Reason.OnShutdown)); BaseContainerManagerTest.WaitForApplicationState(containerManager, cId.GetApplicationAttemptId ().GetApplicationId(), ApplicationState.Finished); // Now ascertain that the resources are localised correctly. foreach (FilePath f_1 in new FilePath[] { appDir, containerDir, appSysDir, containerSysDir }) { // Wait for deletion. Deletion can happen long after AppFinish because of // the async DeletionService int timeout = 0; while (f_1.Exists() && timeout++ < 15) { Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.IsFalse(f_1.GetAbsolutePath() + " exists!!", f_1.Exists()); } // Wait for deletion int timeout_1 = 0; while (targetFile.Exists() && timeout_1++ < 15) { Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.IsFalse(targetFile.GetAbsolutePath() + " exists!!", targetFile .Exists()); }