public NonAggregatingLogHandlerWithMockExecutor(TestNonAggregatingLogHandler _enclosing , Dispatcher dispatcher, DeletionService delService, LocalDirsHandlerService dirsHandler ) : this(dispatcher, delService, dirsHandler, new NMNullStateStoreService()) { this._enclosing = _enclosing; }
public NonAggregatingLogHandlerWithMockExecutor(TestNonAggregatingLogHandler _enclosing , Dispatcher dispatcher, DeletionService delService, LocalDirsHandlerService dirsHandler , NMStateStoreService stateStore) : base(dispatcher, delService, dirsHandler, stateStore) { this._enclosing = _enclosing; }
internal ResourceRetentionSet(DeletionService delService, long targetSize, SortedDictionary <LocalizedResource, LocalResourcesTracker> retain) { this.retain = retain; this.delService = delService; this.targetSize = targetSize; }
/// <summary> /// Function to verify that the DeletionService object received the right /// requests. /// </summary> /// <param name="delService">the DeletionService mock which we verify against</param> /// <param name="user">the user name to use when verifying the deletion</param> /// <param name="timeout"> /// amount in milliseconds to wait before we decide the calls /// didn't come through /// </param> /// <param name="matchPaths">the paths to match in the delete calls</param> /// <exception cref="Org.Mockito.Exceptions.Verification.WantedButNotInvoked">if the calls could not be verified /// </exception> internal static void TestDeletionServiceCall(DeletionService delService, string user , long timeout, params Path[] matchPaths) { long verifyStartTime = Runtime.CurrentTimeMillis(); WantedButNotInvoked notInvokedException = null; bool matched = false; while (!matched && Runtime.CurrentTimeMillis() < verifyStartTime + timeout) { try { Org.Mockito.Mockito.Verify(delService).Delete(Org.Mockito.Matchers.Eq(user), (Path )Org.Mockito.Matchers.Eq(null), Org.Mockito.Mockito.ArgThat(new TestNonAggregatingLogHandler.DeletePathsMatcher (matchPaths))); matched = true; } catch (WantedButNotInvoked e) { notInvokedException = e; try { Sharpen.Thread.Sleep(50l); } catch (Exception) { } } } if (!matched) { throw notInvokedException; } return; }
public virtual void Setup() { localFS.Delete(new Path(localDir.GetAbsolutePath()), true); localFS.Delete(new Path(tmpDir.GetAbsolutePath()), true); localFS.Delete(new Path(localLogDir.GetAbsolutePath()), true); localFS.Delete(new Path(remoteLogDir.GetAbsolutePath()), true); localDir.Mkdir(); tmpDir.Mkdir(); localLogDir.Mkdir(); remoteLogDir.Mkdir(); Log.Info("Created localDir in " + localDir.GetAbsolutePath()); Log.Info("Created tmpDir in " + tmpDir.GetAbsolutePath()); string bindAddress = "0.0.0.0:12345"; conf.Set(YarnConfiguration.NmAddress, bindAddress); conf.Set(YarnConfiguration.NmLocalDirs, localDir.GetAbsolutePath()); conf.Set(YarnConfiguration.NmLogDirs, localLogDir.GetAbsolutePath()); conf.Set(YarnConfiguration.NmRemoteAppLogDir, remoteLogDir.GetAbsolutePath()); conf.SetLong(YarnConfiguration.NmLogRetainSeconds, 1); // Default delSrvc delSrvc = CreateDeletionService(); delSrvc.Init(conf); exec = CreateContainerExecutor(); nodeHealthChecker = new NodeHealthCheckerService(); nodeHealthChecker.Init(conf); dirsHandler = nodeHealthChecker.GetDiskHandler(); containerManager = CreateContainerManager(delSrvc); ((NodeManager.NMContext)context).SetContainerManager(containerManager); nodeStatusUpdater.Init(conf); containerManager.Init(conf); nodeStatusUpdater.Start(); }
public _ContainerManagerImpl_69(TestNMProxy _enclosing, Context baseArg1, ContainerExecutor baseArg2, DeletionService baseArg3, NodeStatusUpdater baseArg4, NodeManagerMetrics baseArg5, ApplicationACLsManager baseArg6, LocalDirsHandlerService baseArg7) : base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7) { this._enclosing = _enclosing; }
public virtual bool Remove(LocalizedResource rem, DeletionService delService) { // current synchronization guaranteed by crude RLS event for cleanup LocalizedResource rsrc = localrsrc[rem.GetRequest()]; if (null == rsrc) { Log.Error("Attempt to remove absent resource: " + rem.GetRequest() + " from " + GetUser ()); return(true); } if (rsrc.GetRefCount() > 0 || ResourceState.Downloading.Equals(rsrc.GetState()) || rsrc != rem) { // internal error Log.Error("Attempt to remove resource: " + rsrc + " with non-zero refcount"); return(false); } else { // ResourceState is LOCALIZED or INIT if (ResourceState.Localized.Equals(rsrc.GetState())) { delService.Delete(GetUser(), GetPathToDelete(rsrc.GetLocalPath())); } RemoveResource(rem.GetRequest()); Log.Info("Removed " + rsrc.GetLocalPath() + " from localized cache"); return(true); } }
public RedirectController(RedirectService redirectService, ILogger <RedirectController> logger, IEnumerable <IHostedService> services) { _redirectService = redirectService; _logger = logger; // Have to do this because we can't directly injected DeletionService _deletionService = (DeletionService)services.FirstOrDefault(w => w.GetType() == typeof(DeletionService)); }
/// <summary> /// Function to run a log handler with directories failing the getFileStatus /// call. /// </summary> /// <remarks> /// Function to run a log handler with directories failing the getFileStatus /// call. The function accepts the log handler, setup the mocks to fail with /// specific exceptions and ensures the deletion service has the correct calls. /// </remarks> /// <param name="logHandler">the logHandler implementation to test</param> /// <param name="appId"> /// the application id that we wish when sending events to the log /// handler /// </param> /// <param name="user">the user name to use</param> /// <param name="mockDelService"> /// a mock of the DeletionService which we will verify /// the delete calls against /// </param> /// <param name="dirsHandler"> /// a spy or mock on the LocalDirsHandler service used to /// when creating the logHandler. It needs to be a spy so that we can intercept /// the getAllLogDirs() call. /// </param> /// <param name="conf">the configuration used</param> /// <param name="spylfs">a spy on the AbstractFileSystem object used when creating lfs /// </param> /// <param name="lfs"> /// the FileContext object to be used to mock the getFileStatus() /// calls /// </param> /// <param name="localLogDirs"> /// list of the log dirs to run the test against, must have /// at least 7 entries /// </param> /// <exception cref="System.Exception"/> public static void RunMockedFailedDirs(LogHandler logHandler, ApplicationId appId , string user, DeletionService mockDelService, LocalDirsHandlerService dirsHandler , Configuration conf, AbstractFileSystem spylfs, FileContext lfs, FilePath[] localLogDirs ) { IDictionary <ApplicationAccessType, string> appAcls = new Dictionary <ApplicationAccessType , string>(); if (localLogDirs.Length < 7) { throw new ArgumentException("Argument localLogDirs must be at least of length 7"); } Path[] localAppLogDirPaths = new Path[localLogDirs.Length]; for (int i = 0; i < localAppLogDirPaths.Length; i++) { localAppLogDirPaths[i] = new Path(localLogDirs[i].GetAbsolutePath(), appId.ToString ()); } IList <string> localLogDirPaths = new AList <string>(localLogDirs.Length); for (int i_1 = 0; i_1 < localLogDirs.Length; i_1++) { localLogDirPaths.AddItem(localLogDirs[i_1].GetAbsolutePath()); } // setup mocks FsPermission defaultPermission = FsPermission.GetDirDefault().ApplyUMask(lfs.GetUMask ()); FileStatus fs = new FileStatus(0, true, 1, 0, Runtime.CurrentTimeMillis(), 0, defaultPermission , string.Empty, string.Empty, new Path(localLogDirs[0].GetAbsolutePath())); Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.IsA <Path>()); Org.Mockito.Mockito.DoReturn(localLogDirPaths).When(dirsHandler).GetLogDirsForCleanup (); logHandler.Handle(new LogHandlerAppStartedEvent(appId, user, null, ContainerLogsRetentionPolicy .AllContainers, appAcls)); // test case where some dirs have the log dir to delete // mock some dirs throwing various exceptions // verify deletion happens only on the others Org.Mockito.Mockito.DoThrow(new FileNotFoundException()).When(spylfs).GetFileStatus (Matchers.Eq(localAppLogDirPaths[0])); Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.Eq(localAppLogDirPaths [1])); Org.Mockito.Mockito.DoThrow(new AccessControlException()).When(spylfs).GetFileStatus (Matchers.Eq(localAppLogDirPaths[2])); Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.Eq(localAppLogDirPaths [3])); Org.Mockito.Mockito.DoThrow(new IOException()).When(spylfs).GetFileStatus(Matchers.Eq (localAppLogDirPaths[4])); Org.Mockito.Mockito.DoThrow(new UnsupportedFileSystemException("test")).When(spylfs ).GetFileStatus(Matchers.Eq(localAppLogDirPaths[5])); Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.Eq(localAppLogDirPaths [6])); logHandler.Handle(new LogHandlerAppFinishedEvent(appId)); TestDeletionServiceCall(mockDelService, user, 5000, localAppLogDirPaths[1], localAppLogDirPaths [3], localAppLogDirPaths[6]); return; }
/// <returns> /// /// <see cref="Org.Apache.Hadoop.FS.Path"/> /// absolute path for localization which includes local /// directory path and the relative hierarchical path (if use local /// cache directory manager is enabled) /// </returns> /// <?/> /// <?/> /// <?/> public virtual Path GetPathForLocalization(LocalResourceRequest req, Path localDirPath , DeletionService delService) { Path rPath = localDirPath; if (useLocalCacheDirectoryManager && localDirPath != null) { if (!directoryManagers.Contains(localDirPath)) { directoryManagers.PutIfAbsent(localDirPath, new LocalCacheDirectoryManager(conf)); } LocalCacheDirectoryManager dir = directoryManagers[localDirPath]; rPath = localDirPath; string hierarchicalPath = dir.GetRelativePathForLocalization(); // For most of the scenarios we will get root path only which // is an empty string if (!hierarchicalPath.IsEmpty()) { rPath = new Path(localDirPath, hierarchicalPath); } inProgressLocalResourcesMap[req] = rPath; } while (true) { Path uniquePath = new Path(rPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet ())); FilePath file = new FilePath(uniquePath.ToUri().GetRawPath()); if (!file.Exists()) { rPath = uniquePath; break; } // If the directory already exists, delete it and move to next one. Log.Warn("Directory " + uniquePath + " already exists, " + "try next one."); if (delService != null) { delService.Delete(GetUser(), uniquePath); } } Path localPath = new Path(rPath, req.GetPath().GetName()); LocalizedResource rsrc = localrsrc[req]; rsrc.SetLocalPath(localPath); LocalResource lr = LocalResource.NewInstance(req.GetResource(), req.GetType(), req .GetVisibility(), req.GetSize(), req.GetTimestamp()); try { stateStore.StartResourceLocalization(user, appId, ((LocalResourcePBImpl)lr).GetProto (), localPath); } catch (IOException e) { Log.Error("Unable to record localization start for " + rsrc, e); } return(rPath); }
public _ContainerManagerImpl_319(LogHandler logHandler, ResourceLocalizationService rsrcSrv, ContainersLauncher launcher, Context baseArg1, ContainerExecutor baseArg2 , DeletionService baseArg3, NodeStatusUpdater baseArg4, NodeManagerMetrics baseArg5 , ApplicationACLsManager baseArg6, LocalDirsHandlerService baseArg7) : base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7) { this.logHandler = logHandler; this.rsrcSrv = rsrcSrv; this.launcher = launcher; }
public NonAggregatingLogHandler(Dispatcher dispatcher, DeletionService delService , LocalDirsHandlerService dirsHandler, NMStateStoreService stateStore) : base(typeof(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Loghandler.NonAggregatingLogHandler ).FullName) { this.dispatcher = dispatcher; this.delService = delService; this.dirsHandler = dirsHandler; this.stateStore = stateStore; this.appOwners = new ConcurrentHashMap <ApplicationId, string>(); }
public virtual void Setup() { mockDelService = Org.Mockito.Mockito.Mock <DeletionService>(); conf = new YarnConfiguration(); dispatcher = CreateDispatcher(conf); appEventHandler = Org.Mockito.Mockito.Mock <EventHandler>(); dispatcher.Register(typeof(ApplicationEventType), appEventHandler); appId = BuilderUtils.NewApplicationId(1234, 1); appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1); container11 = BuilderUtils.NewContainerId(appAttemptId, 1); dirsHandler = new LocalDirsHandlerService(); }
public EventManagementController( ISearchService searchService, DeletionService deletionService, CreationService creationService, EditService editService, IHostingEnvironment hostingEnvironment) { _searchService = searchService; _deletionService = deletionService; _creationService = creationService; _editService = editService; _hostingEnvironment = hostingEnvironment; }
public virtual void TestHandlingApplicationFinishedEvent() { DeletionService delService = new DeletionService(null); NonAggregatingLogHandler aggregatingLogHandler = new NonAggregatingLogHandler(new InlineDispatcher(), delService, dirsHandler, new NMNullStateStoreService()); dirsHandler.Init(conf); dirsHandler.Start(); delService.Init(conf); delService.Start(); aggregatingLogHandler.Init(conf); aggregatingLogHandler.Start(); // It should NOT throw RejectedExecutionException aggregatingLogHandler.Handle(new LogHandlerAppFinishedEvent(appId)); aggregatingLogHandler.Stop(); // It should NOT throw RejectedExecutionException after stopping // handler service. aggregatingLogHandler.Handle(new LogHandlerAppFinishedEvent(appId)); aggregatingLogHandler.Close(); }
public LogAggregationService(Dispatcher dispatcher, Context context, DeletionService deletionService, LocalDirsHandlerService dirsHandler) : base(typeof(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Logaggregation.LogAggregationService ).FullName) { /* * Expected deployment TLD will be 1777, owner=<NMOwner>, group=<NMGroup - * Group to which NMOwner belongs> App dirs will be created as 770, * owner=<AppOwner>, group=<NMGroup>: so that the owner and <NMOwner> can * access / modify the files. * <NMGroup> should obviously be a limited access group. */ this.dispatcher = dispatcher; this.context = context; this.deletionService = deletionService; this.dirsHandler = dirsHandler; this.appLogAggregators = new ConcurrentHashMap <ApplicationId, AppLogAggregator>(); this.threadPool = Executors.NewCachedThreadPool(new ThreadFactoryBuilder().SetNameFormat ("LogAggregationService #%d").Build()); }
public virtual void TestRsrcUnused() { DeletionService delService = Org.Mockito.Mockito.Mock <DeletionService>(); long TargetMb = 10 << 20; ResourceRetentionSet rss = new ResourceRetentionSet(delService, TargetMb); // 3MB files @{10, 15} LocalResourcesTracker pubTracker = CreateMockTracker(null, 3 * 1024 * 1024, 2, 10 , 5); // 1MB files @{3, 6, 9, 12} LocalResourcesTracker trackerA = CreateMockTracker("A", 1 * 1024 * 1024, 4, 3, 3); // 4MB file @{1} LocalResourcesTracker trackerB = CreateMockTracker("B", 4 * 1024 * 1024, 1, 10, 5 ); // 2MB files @{7, 9, 11} LocalResourcesTracker trackerC = CreateMockTracker("C", 2 * 1024 * 1024, 3, 7, 2); // Total cache: 20MB; verify removed at least 10MB rss.AddResources(pubTracker); rss.AddResources(trackerA); rss.AddResources(trackerB); rss.AddResources(trackerC); long deleted = 0L; ArgumentCaptor <LocalizedResource> captor = ArgumentCaptor.ForClass <LocalizedResource >(); Org.Mockito.Mockito.Verify(pubTracker, Org.Mockito.Mockito.AtMost(2)).Remove(captor .Capture(), IsA <DeletionService>()); Org.Mockito.Mockito.Verify(trackerA, Org.Mockito.Mockito.AtMost(4)).Remove(captor .Capture(), IsA <DeletionService>()); Org.Mockito.Mockito.Verify(trackerB, Org.Mockito.Mockito.AtMost(1)).Remove(captor .Capture(), IsA <DeletionService>()); Org.Mockito.Mockito.Verify(trackerC, Org.Mockito.Mockito.AtMost(3)).Remove(captor .Capture(), IsA <DeletionService>()); foreach (LocalizedResource rem in captor.GetAllValues()) { deleted += rem.GetSize(); } NUnit.Framework.Assert.IsTrue(deleted >= 10 * 1024 * 1024); NUnit.Framework.Assert.IsTrue(deleted < 15 * 1024 * 1024); }
public _ResourceLocalizationService_287(Dispatcher baseArg1, ContainerExecutor baseArg2 , DeletionService baseArg3, LocalDirsHandlerService baseArg4, Context baseArg5) : base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5) { }
public _ContainerManagerImpl_191(Context baseArg1, ContainerExecutor baseArg2, DeletionService baseArg3, NodeStatusUpdater baseArg4, NodeManagerMetrics baseArg5, ApplicationACLsManager baseArg6, LocalDirsHandlerService baseArg7) : base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7) { }
protected internal virtual ContainerManagerImpl CreateContainerManager(DeletionService delSrvc) { return(new _ContainerManagerImpl_191(context, exec, delSrvc, nodeStatusUpdater, metrics , new ApplicationACLsManager(conf), dirsHandler)); }
internal ResourceRetentionSet(DeletionService delService, long targetSize) : this(delService, targetSize, new ResourceRetentionSet.LRUComparator()) { }
public virtual void TestLocalFilesCleanup() { // Real del service delSrvc = new DeletionService(exec); delSrvc.Init(conf); containerManager = CreateContainerManager(delSrvc); containerManager.Init(conf); containerManager.Start(); // ////// Create the resources for the container FilePath dir = new FilePath(tmpDir, "dir"); dir.Mkdirs(); FilePath file = new FilePath(dir, "file"); PrintWriter fileWriter = new PrintWriter(file); fileWriter.Write("Hello World!"); fileWriter.Close(); // ////// Construct the Container-id ContainerId cId = CreateContainerId(0); ApplicationId appId = cId.GetApplicationAttemptId().GetApplicationId(); // ////// Construct the container-spec. ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); // containerLaunchContext.resources = // new HashMap<CharSequence, LocalResource>(); URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(FileContext.GetLocalFSFileContext ().MakeQualified(new Path(file.GetAbsolutePath()))); LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>(); rsrc_alpha.SetResource(resource_alpha); rsrc_alpha.SetSize(-1); rsrc_alpha.SetVisibility(LocalResourceVisibility.Application); rsrc_alpha.SetType(LocalResourceType.File); rsrc_alpha.SetTimestamp(file.LastModified()); string destinationFile = "dest_file"; IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); localResources[destinationFile] = rsrc_alpha; containerLaunchContext.SetLocalResources(localResources); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context .GetContainerTokenSecretManager())); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); containerManager.StartContainers(allRequests); BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState .Complete); BaseContainerManagerTest.WaitForApplicationState(containerManager, cId.GetApplicationAttemptId ().GetApplicationId(), ApplicationState.Running); // Now ascertain that the resources are localised correctly. string appIDStr = ConverterUtils.ToString(appId); string containerIDStr = ConverterUtils.ToString(cId); FilePath userCacheDir = new FilePath(localDir, ContainerLocalizer.Usercache); FilePath userDir = new FilePath(userCacheDir, user); FilePath appCache = new FilePath(userDir, ContainerLocalizer.Appcache); FilePath appDir = new FilePath(appCache, appIDStr); FilePath containerDir = new FilePath(appDir, containerIDStr); FilePath targetFile = new FilePath(containerDir, destinationFile); FilePath sysDir = new FilePath(localDir, ResourceLocalizationService.NmPrivateDir ); FilePath appSysDir = new FilePath(sysDir, appIDStr); FilePath containerSysDir = new FilePath(appSysDir, containerIDStr); // AppDir should still exist NUnit.Framework.Assert.IsTrue("AppDir " + appDir.GetAbsolutePath() + " doesn't exist!!" , appDir.Exists()); NUnit.Framework.Assert.IsTrue("AppSysDir " + appSysDir.GetAbsolutePath() + " doesn't exist!!" , appSysDir.Exists()); foreach (FilePath f in new FilePath[] { containerDir, containerSysDir }) { NUnit.Framework.Assert.IsFalse(f.GetAbsolutePath() + " exists!!", f.Exists()); } NUnit.Framework.Assert.IsFalse(targetFile.GetAbsolutePath() + " exists!!", targetFile .Exists()); // Simulate RM sending an AppFinish event. containerManager.Handle(new CMgrCompletedAppsEvent(Arrays.AsList(new ApplicationId [] { appId }), CMgrCompletedAppsEvent.Reason.OnShutdown)); BaseContainerManagerTest.WaitForApplicationState(containerManager, cId.GetApplicationAttemptId ().GetApplicationId(), ApplicationState.Finished); // Now ascertain that the resources are localised correctly. foreach (FilePath f_1 in new FilePath[] { appDir, containerDir, appSysDir, containerSysDir }) { // Wait for deletion. Deletion can happen long after AppFinish because of // the async DeletionService int timeout = 0; while (f_1.Exists() && timeout++ < 15) { Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.IsFalse(f_1.GetAbsolutePath() + " exists!!", f_1.Exists()); } // Wait for deletion int timeout_1 = 0; while (targetFile.Exists() && timeout_1++ < 15) { Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.IsFalse(targetFile.GetAbsolutePath() + " exists!!", targetFile .Exists()); }
internal ResourceRetentionSet(DeletionService delService, long targetSize, IComparer <LocalizedResource> cmp) : this(delService, targetSize, new SortedDictionary <LocalizedResource, LocalResourcesTracker >(cmp)) { }
protected internal override LogHandler CreateLogHandler(Configuration conf, Context context, DeletionService deletionService) { return(logHandler); }
protected internal override ResourceLocalizationService CreateResourceLocalizationService (ContainerExecutor exec, DeletionService deletionContext, Context context) { return(rsrcSrv); }
public AppLogAggregatorImpl(Dispatcher dispatcher, DeletionService deletionService , Configuration conf, ApplicationId appId, UserGroupInformation userUgi, NodeId nodeId, LocalDirsHandlerService dirsHandler, Path remoteNodeLogFileForApp, ContainerLogsRetentionPolicy retentionPolicy, IDictionary <ApplicationAccessType, string> appAcls, LogAggregationContext logAggregationContext, Context context, FileContext lfs) { // This is temporary solution. The configuration will be deleted once // we find a more scalable method to only write a single log file per LRS. // This configuration is for debug and test purpose. By setting // this configuration as true. We can break the lower bound of // NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS. // This variable is only for testing this.dispatcher = dispatcher; this.conf = conf; this.delService = deletionService; this.appId = appId; this.applicationId = ConverterUtils.ToString(appId); this.userUgi = userUgi; this.dirsHandler = dirsHandler; this.remoteNodeLogFileForApp = remoteNodeLogFileForApp; this.remoteNodeTmpLogFileForApp = GetRemoteNodeTmpLogFileForApp(); this.retentionPolicy = retentionPolicy; this.pendingContainers = new LinkedBlockingQueue <ContainerId>(); this.appAcls = appAcls; this.lfs = lfs; this.logAggregationContext = logAggregationContext; this.context = context; this.nodeId = nodeId; int configuredRentionSize = conf.GetInt(NmLogAggregationNumLogFilesSizePerApp, DefaultNmLogAggregationNumLogFilesSizePerApp ); if (configuredRentionSize <= 0) { this.retentionSize = DefaultNmLogAggregationNumLogFilesSizePerApp; } else { this.retentionSize = configuredRentionSize; } long configuredRollingMonitorInterval = conf.GetLong(YarnConfiguration.NmLogAggregationRollMonitoringIntervalSeconds , YarnConfiguration.DefaultNmLogAggregationRollMonitoringIntervalSeconds); bool debug_mode = conf.GetBoolean(NmLogAggregationDebugEnabled, DefaultNmLogAggregationDebugEnabled ); if (configuredRollingMonitorInterval > 0 && configuredRollingMonitorInterval < NmLogAggregationMinRollMonitoringIntervalSeconds) { if (debug_mode) { this.rollingMonitorInterval = configuredRollingMonitorInterval; } else { Log.Warn("rollingMonitorIntervall should be more than or equal to " + NmLogAggregationMinRollMonitoringIntervalSeconds + " seconds. Using " + NmLogAggregationMinRollMonitoringIntervalSeconds + " seconds instead." ); this.rollingMonitorInterval = NmLogAggregationMinRollMonitoringIntervalSeconds; } } else { if (configuredRollingMonitorInterval <= 0) { Log.Warn("rollingMonitorInterval is set as " + configuredRollingMonitorInterval + ". " + "The log rolling mornitoring interval is disabled. " + "The logs will be aggregated after this application is finished." ); } else { Log.Warn("rollingMonitorInterval is set as " + configuredRollingMonitorInterval + ". " + "The logs will be aggregated every " + configuredRollingMonitorInterval + " seconds"); } this.rollingMonitorInterval = configuredRollingMonitorInterval; } this.logAggregationInRolling = this.rollingMonitorInterval <= 0 || this.logAggregationContext == null || this.logAggregationContext.GetRolledLogsIncludePattern() == null || this.logAggregationContext.GetRolledLogsIncludePattern().IsEmpty() ? false : true; }