public NonAggregatingLogHandlerWithMockExecutor(TestNonAggregatingLogHandler _enclosing
, Dispatcher dispatcher, DeletionService delService, LocalDirsHandlerService dirsHandler
)
: this(dispatcher, delService, dirsHandler, new NMNullStateStoreService())
{
this._enclosing = _enclosing;
}
public NonAggregatingLogHandlerWithMockExecutor(TestNonAggregatingLogHandler _enclosing
, Dispatcher dispatcher, DeletionService delService, LocalDirsHandlerService dirsHandler
, NMStateStoreService stateStore)
: base(dispatcher, delService, dirsHandler, stateStore)
{
this._enclosing = _enclosing;
}
internal ResourceRetentionSet(DeletionService delService, long targetSize, SortedDictionary
<LocalizedResource, LocalResourcesTracker> retain)
{
this.retain = retain;
this.delService = delService;
this.targetSize = targetSize;
}
/// <summary>
/// Function to verify that the DeletionService object received the right
/// requests.
/// </summary>
/// <param name="delService">the DeletionService mock which we verify against</param>
/// <param name="user">the user name to use when verifying the deletion</param>
/// <param name="timeout">
/// amount in milliseconds to wait before we decide the calls
/// didn't come through
/// </param>
/// <param name="matchPaths">the paths to match in the delete calls</param>
/// <exception cref="Org.Mockito.Exceptions.Verification.WantedButNotInvoked">if the calls could not be verified
/// </exception>
internal static void TestDeletionServiceCall(DeletionService delService, string user
, long timeout, params Path[] matchPaths)
{
long verifyStartTime = Runtime.CurrentTimeMillis();
WantedButNotInvoked notInvokedException = null;
bool matched = false;
while (!matched && Runtime.CurrentTimeMillis() < verifyStartTime + timeout)
{
try
{
Org.Mockito.Mockito.Verify(delService).Delete(Org.Mockito.Matchers.Eq(user), (Path
)Org.Mockito.Matchers.Eq(null), Org.Mockito.Mockito.ArgThat(new TestNonAggregatingLogHandler.DeletePathsMatcher
(matchPaths)));
matched = true;
}
catch (WantedButNotInvoked e)
{
notInvokedException = e;
try
{
Sharpen.Thread.Sleep(50l);
}
catch (Exception)
{
}
}
}
if (!matched)
{
throw notInvokedException;
}
return;
}
public virtual void Setup()
{
localFS.Delete(new Path(localDir.GetAbsolutePath()), true);
localFS.Delete(new Path(tmpDir.GetAbsolutePath()), true);
localFS.Delete(new Path(localLogDir.GetAbsolutePath()), true);
localFS.Delete(new Path(remoteLogDir.GetAbsolutePath()), true);
localDir.Mkdir();
tmpDir.Mkdir();
localLogDir.Mkdir();
remoteLogDir.Mkdir();
Log.Info("Created localDir in " + localDir.GetAbsolutePath());
Log.Info("Created tmpDir in " + tmpDir.GetAbsolutePath());
string bindAddress = "0.0.0.0:12345";
conf.Set(YarnConfiguration.NmAddress, bindAddress);
conf.Set(YarnConfiguration.NmLocalDirs, localDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmLogDirs, localLogDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmRemoteAppLogDir, remoteLogDir.GetAbsolutePath());
conf.SetLong(YarnConfiguration.NmLogRetainSeconds, 1);
// Default delSrvc
delSrvc = CreateDeletionService();
delSrvc.Init(conf);
exec = CreateContainerExecutor();
nodeHealthChecker = new NodeHealthCheckerService();
nodeHealthChecker.Init(conf);
dirsHandler = nodeHealthChecker.GetDiskHandler();
containerManager = CreateContainerManager(delSrvc);
((NodeManager.NMContext)context).SetContainerManager(containerManager);
nodeStatusUpdater.Init(conf);
containerManager.Init(conf);
nodeStatusUpdater.Start();
}
public _ContainerManagerImpl_69(TestNMProxy _enclosing, Context baseArg1, ContainerExecutor
baseArg2, DeletionService baseArg3, NodeStatusUpdater baseArg4, NodeManagerMetrics
baseArg5, ApplicationACLsManager baseArg6, LocalDirsHandlerService baseArg7)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7)
{
this._enclosing = _enclosing;
}
public virtual bool Remove(LocalizedResource rem, DeletionService delService)
{
// current synchronization guaranteed by crude RLS event for cleanup
LocalizedResource rsrc = localrsrc[rem.GetRequest()];
if (null == rsrc)
{
Log.Error("Attempt to remove absent resource: " + rem.GetRequest() + " from " + GetUser
());
return(true);
}
if (rsrc.GetRefCount() > 0 || ResourceState.Downloading.Equals(rsrc.GetState()) ||
rsrc != rem)
{
// internal error
Log.Error("Attempt to remove resource: " + rsrc + " with non-zero refcount");
return(false);
}
else
{
// ResourceState is LOCALIZED or INIT
if (ResourceState.Localized.Equals(rsrc.GetState()))
{
delService.Delete(GetUser(), GetPathToDelete(rsrc.GetLocalPath()));
}
RemoveResource(rem.GetRequest());
Log.Info("Removed " + rsrc.GetLocalPath() + " from localized cache");
return(true);
}
}
public RedirectController(RedirectService redirectService, ILogger <RedirectController> logger,
IEnumerable <IHostedService> services)
{
_redirectService = redirectService;
_logger = logger;
// Have to do this because we can't directly injected DeletionService
_deletionService = (DeletionService)services.FirstOrDefault(w => w.GetType() == typeof(DeletionService));
}
/// <summary>
/// Function to run a log handler with directories failing the getFileStatus
/// call.
/// </summary>
/// <remarks>
/// Function to run a log handler with directories failing the getFileStatus
/// call. The function accepts the log handler, setup the mocks to fail with
/// specific exceptions and ensures the deletion service has the correct calls.
/// </remarks>
/// <param name="logHandler">the logHandler implementation to test</param>
/// <param name="appId">
/// the application id that we wish when sending events to the log
/// handler
/// </param>
/// <param name="user">the user name to use</param>
/// <param name="mockDelService">
/// a mock of the DeletionService which we will verify
/// the delete calls against
/// </param>
/// <param name="dirsHandler">
/// a spy or mock on the LocalDirsHandler service used to
/// when creating the logHandler. It needs to be a spy so that we can intercept
/// the getAllLogDirs() call.
/// </param>
/// <param name="conf">the configuration used</param>
/// <param name="spylfs">a spy on the AbstractFileSystem object used when creating lfs
/// </param>
/// <param name="lfs">
/// the FileContext object to be used to mock the getFileStatus()
/// calls
/// </param>
/// <param name="localLogDirs">
/// list of the log dirs to run the test against, must have
/// at least 7 entries
/// </param>
/// <exception cref="System.Exception"/>
public static void RunMockedFailedDirs(LogHandler logHandler, ApplicationId appId
, string user, DeletionService mockDelService, LocalDirsHandlerService dirsHandler
, Configuration conf, AbstractFileSystem spylfs, FileContext lfs, FilePath[] localLogDirs
)
{
IDictionary <ApplicationAccessType, string> appAcls = new Dictionary <ApplicationAccessType
, string>();
if (localLogDirs.Length < 7)
{
throw new ArgumentException("Argument localLogDirs must be at least of length 7");
}
Path[] localAppLogDirPaths = new Path[localLogDirs.Length];
for (int i = 0; i < localAppLogDirPaths.Length; i++)
{
localAppLogDirPaths[i] = new Path(localLogDirs[i].GetAbsolutePath(), appId.ToString
());
}
IList <string> localLogDirPaths = new AList <string>(localLogDirs.Length);
for (int i_1 = 0; i_1 < localLogDirs.Length; i_1++)
{
localLogDirPaths.AddItem(localLogDirs[i_1].GetAbsolutePath());
}
// setup mocks
FsPermission defaultPermission = FsPermission.GetDirDefault().ApplyUMask(lfs.GetUMask
());
FileStatus fs = new FileStatus(0, true, 1, 0, Runtime.CurrentTimeMillis(), 0, defaultPermission
, string.Empty, string.Empty, new Path(localLogDirs[0].GetAbsolutePath()));
Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.IsA <Path>());
Org.Mockito.Mockito.DoReturn(localLogDirPaths).When(dirsHandler).GetLogDirsForCleanup
();
logHandler.Handle(new LogHandlerAppStartedEvent(appId, user, null, ContainerLogsRetentionPolicy
.AllContainers, appAcls));
// test case where some dirs have the log dir to delete
// mock some dirs throwing various exceptions
// verify deletion happens only on the others
Org.Mockito.Mockito.DoThrow(new FileNotFoundException()).When(spylfs).GetFileStatus
(Matchers.Eq(localAppLogDirPaths[0]));
Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.Eq(localAppLogDirPaths
[1]));
Org.Mockito.Mockito.DoThrow(new AccessControlException()).When(spylfs).GetFileStatus
(Matchers.Eq(localAppLogDirPaths[2]));
Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.Eq(localAppLogDirPaths
[3]));
Org.Mockito.Mockito.DoThrow(new IOException()).When(spylfs).GetFileStatus(Matchers.Eq
(localAppLogDirPaths[4]));
Org.Mockito.Mockito.DoThrow(new UnsupportedFileSystemException("test")).When(spylfs
).GetFileStatus(Matchers.Eq(localAppLogDirPaths[5]));
Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.Eq(localAppLogDirPaths
[6]));
logHandler.Handle(new LogHandlerAppFinishedEvent(appId));
TestDeletionServiceCall(mockDelService, user, 5000, localAppLogDirPaths[1], localAppLogDirPaths
[3], localAppLogDirPaths[6]);
return;
}
/// <returns>
///
/// <see cref="Org.Apache.Hadoop.FS.Path"/>
/// absolute path for localization which includes local
/// directory path and the relative hierarchical path (if use local
/// cache directory manager is enabled)
/// </returns>
/// <?/>
/// <?/>
/// <?/>
public virtual Path GetPathForLocalization(LocalResourceRequest req, Path localDirPath
, DeletionService delService)
{
Path rPath = localDirPath;
if (useLocalCacheDirectoryManager && localDirPath != null)
{
if (!directoryManagers.Contains(localDirPath))
{
directoryManagers.PutIfAbsent(localDirPath, new LocalCacheDirectoryManager(conf));
}
LocalCacheDirectoryManager dir = directoryManagers[localDirPath];
rPath = localDirPath;
string hierarchicalPath = dir.GetRelativePathForLocalization();
// For most of the scenarios we will get root path only which
// is an empty string
if (!hierarchicalPath.IsEmpty())
{
rPath = new Path(localDirPath, hierarchicalPath);
}
inProgressLocalResourcesMap[req] = rPath;
}
while (true)
{
Path uniquePath = new Path(rPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
()));
FilePath file = new FilePath(uniquePath.ToUri().GetRawPath());
if (!file.Exists())
{
rPath = uniquePath;
break;
}
// If the directory already exists, delete it and move to next one.
Log.Warn("Directory " + uniquePath + " already exists, " + "try next one.");
if (delService != null)
{
delService.Delete(GetUser(), uniquePath);
}
}
Path localPath = new Path(rPath, req.GetPath().GetName());
LocalizedResource rsrc = localrsrc[req];
rsrc.SetLocalPath(localPath);
LocalResource lr = LocalResource.NewInstance(req.GetResource(), req.GetType(), req
.GetVisibility(), req.GetSize(), req.GetTimestamp());
try
{
stateStore.StartResourceLocalization(user, appId, ((LocalResourcePBImpl)lr).GetProto
(), localPath);
}
catch (IOException e)
{
Log.Error("Unable to record localization start for " + rsrc, e);
}
return(rPath);
}
public _ContainerManagerImpl_319(LogHandler logHandler, ResourceLocalizationService
rsrcSrv, ContainersLauncher launcher, Context baseArg1, ContainerExecutor baseArg2
, DeletionService baseArg3, NodeStatusUpdater baseArg4, NodeManagerMetrics baseArg5
, ApplicationACLsManager baseArg6, LocalDirsHandlerService baseArg7)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7)
{
this.logHandler = logHandler;
this.rsrcSrv = rsrcSrv;
this.launcher = launcher;
}
public NonAggregatingLogHandler(Dispatcher dispatcher, DeletionService delService
, LocalDirsHandlerService dirsHandler, NMStateStoreService stateStore)
: base(typeof(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Loghandler.NonAggregatingLogHandler
).FullName)
{
this.dispatcher = dispatcher;
this.delService = delService;
this.dirsHandler = dirsHandler;
this.stateStore = stateStore;
this.appOwners = new ConcurrentHashMap <ApplicationId, string>();
}
public virtual void Setup()
{
mockDelService = Org.Mockito.Mockito.Mock <DeletionService>();
conf = new YarnConfiguration();
dispatcher = CreateDispatcher(conf);
appEventHandler = Org.Mockito.Mockito.Mock <EventHandler>();
dispatcher.Register(typeof(ApplicationEventType), appEventHandler);
appId = BuilderUtils.NewApplicationId(1234, 1);
appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1);
container11 = BuilderUtils.NewContainerId(appAttemptId, 1);
dirsHandler = new LocalDirsHandlerService();
}
public EventManagementController(
ISearchService searchService,
DeletionService deletionService,
CreationService creationService,
EditService editService,
IHostingEnvironment hostingEnvironment)
{
_searchService = searchService;
_deletionService = deletionService;
_creationService = creationService;
_editService = editService;
_hostingEnvironment = hostingEnvironment;
}
public virtual void TestHandlingApplicationFinishedEvent()
{
DeletionService delService = new DeletionService(null);
NonAggregatingLogHandler aggregatingLogHandler = new NonAggregatingLogHandler(new
InlineDispatcher(), delService, dirsHandler, new NMNullStateStoreService());
dirsHandler.Init(conf);
dirsHandler.Start();
delService.Init(conf);
delService.Start();
aggregatingLogHandler.Init(conf);
aggregatingLogHandler.Start();
// It should NOT throw RejectedExecutionException
aggregatingLogHandler.Handle(new LogHandlerAppFinishedEvent(appId));
aggregatingLogHandler.Stop();
// It should NOT throw RejectedExecutionException after stopping
// handler service.
aggregatingLogHandler.Handle(new LogHandlerAppFinishedEvent(appId));
aggregatingLogHandler.Close();
}
public LogAggregationService(Dispatcher dispatcher, Context context, DeletionService
deletionService, LocalDirsHandlerService dirsHandler)
: base(typeof(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Logaggregation.LogAggregationService
).FullName)
{
/*
* Expected deployment TLD will be 1777, owner=<NMOwner>, group=<NMGroup -
* Group to which NMOwner belongs> App dirs will be created as 770,
* owner=<AppOwner>, group=<NMGroup>: so that the owner and <NMOwner> can
* access / modify the files.
* <NMGroup> should obviously be a limited access group.
*/
this.dispatcher = dispatcher;
this.context = context;
this.deletionService = deletionService;
this.dirsHandler = dirsHandler;
this.appLogAggregators = new ConcurrentHashMap <ApplicationId, AppLogAggregator>();
this.threadPool = Executors.NewCachedThreadPool(new ThreadFactoryBuilder().SetNameFormat
("LogAggregationService #%d").Build());
}
public virtual void TestRsrcUnused()
{
DeletionService delService = Org.Mockito.Mockito.Mock <DeletionService>();
long TargetMb = 10 << 20;
ResourceRetentionSet rss = new ResourceRetentionSet(delService, TargetMb);
// 3MB files @{10, 15}
LocalResourcesTracker pubTracker = CreateMockTracker(null, 3 * 1024 * 1024, 2, 10
, 5);
// 1MB files @{3, 6, 9, 12}
LocalResourcesTracker trackerA = CreateMockTracker("A", 1 * 1024 * 1024, 4, 3, 3);
// 4MB file @{1}
LocalResourcesTracker trackerB = CreateMockTracker("B", 4 * 1024 * 1024, 1, 10, 5
);
// 2MB files @{7, 9, 11}
LocalResourcesTracker trackerC = CreateMockTracker("C", 2 * 1024 * 1024, 3, 7, 2);
// Total cache: 20MB; verify removed at least 10MB
rss.AddResources(pubTracker);
rss.AddResources(trackerA);
rss.AddResources(trackerB);
rss.AddResources(trackerC);
long deleted = 0L;
ArgumentCaptor <LocalizedResource> captor = ArgumentCaptor.ForClass <LocalizedResource
>();
Org.Mockito.Mockito.Verify(pubTracker, Org.Mockito.Mockito.AtMost(2)).Remove(captor
.Capture(), IsA <DeletionService>());
Org.Mockito.Mockito.Verify(trackerA, Org.Mockito.Mockito.AtMost(4)).Remove(captor
.Capture(), IsA <DeletionService>());
Org.Mockito.Mockito.Verify(trackerB, Org.Mockito.Mockito.AtMost(1)).Remove(captor
.Capture(), IsA <DeletionService>());
Org.Mockito.Mockito.Verify(trackerC, Org.Mockito.Mockito.AtMost(3)).Remove(captor
.Capture(), IsA <DeletionService>());
foreach (LocalizedResource rem in captor.GetAllValues())
{
deleted += rem.GetSize();
}
NUnit.Framework.Assert.IsTrue(deleted >= 10 * 1024 * 1024);
NUnit.Framework.Assert.IsTrue(deleted < 15 * 1024 * 1024);
}
public _ResourceLocalizationService_287(Dispatcher baseArg1, ContainerExecutor baseArg2
, DeletionService baseArg3, LocalDirsHandlerService baseArg4, Context baseArg5)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5)
{
}
public _ContainerManagerImpl_191(Context baseArg1, ContainerExecutor baseArg2, DeletionService
baseArg3, NodeStatusUpdater baseArg4, NodeManagerMetrics baseArg5, ApplicationACLsManager
baseArg6, LocalDirsHandlerService baseArg7)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7)
{
}
protected internal virtual ContainerManagerImpl CreateContainerManager(DeletionService
delSrvc)
{
return(new _ContainerManagerImpl_191(context, exec, delSrvc, nodeStatusUpdater, metrics
, new ApplicationACLsManager(conf), dirsHandler));
}
internal ResourceRetentionSet(DeletionService delService, long targetSize)
: this(delService, targetSize, new ResourceRetentionSet.LRUComparator())
{
}
public virtual void TestLocalFilesCleanup()
{
// Real del service
delSrvc = new DeletionService(exec);
delSrvc.Init(conf);
containerManager = CreateContainerManager(delSrvc);
containerManager.Init(conf);
containerManager.Start();
// ////// Create the resources for the container
FilePath dir = new FilePath(tmpDir, "dir");
dir.Mkdirs();
FilePath file = new FilePath(dir, "file");
PrintWriter fileWriter = new PrintWriter(file);
fileWriter.Write("Hello World!");
fileWriter.Close();
// ////// Construct the Container-id
ContainerId cId = CreateContainerId(0);
ApplicationId appId = cId.GetApplicationAttemptId().GetApplicationId();
// ////// Construct the container-spec.
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
// containerLaunchContext.resources =
// new HashMap<CharSequence, LocalResource>();
URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(FileContext.GetLocalFSFileContext
().MakeQualified(new Path(file.GetAbsolutePath())));
LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();
rsrc_alpha.SetResource(resource_alpha);
rsrc_alpha.SetSize(-1);
rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
rsrc_alpha.SetType(LocalResourceType.File);
rsrc_alpha.SetTimestamp(file.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = rsrc_alpha;
containerLaunchContext.SetLocalResources(localResources);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context
.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
.Complete);
BaseContainerManagerTest.WaitForApplicationState(containerManager, cId.GetApplicationAttemptId
().GetApplicationId(), ApplicationState.Running);
// Now ascertain that the resources are localised correctly.
string appIDStr = ConverterUtils.ToString(appId);
string containerIDStr = ConverterUtils.ToString(cId);
FilePath userCacheDir = new FilePath(localDir, ContainerLocalizer.Usercache);
FilePath userDir = new FilePath(userCacheDir, user);
FilePath appCache = new FilePath(userDir, ContainerLocalizer.Appcache);
FilePath appDir = new FilePath(appCache, appIDStr);
FilePath containerDir = new FilePath(appDir, containerIDStr);
FilePath targetFile = new FilePath(containerDir, destinationFile);
FilePath sysDir = new FilePath(localDir, ResourceLocalizationService.NmPrivateDir
);
FilePath appSysDir = new FilePath(sysDir, appIDStr);
FilePath containerSysDir = new FilePath(appSysDir, containerIDStr);
// AppDir should still exist
NUnit.Framework.Assert.IsTrue("AppDir " + appDir.GetAbsolutePath() + " doesn't exist!!"
, appDir.Exists());
NUnit.Framework.Assert.IsTrue("AppSysDir " + appSysDir.GetAbsolutePath() + " doesn't exist!!"
, appSysDir.Exists());
foreach (FilePath f in new FilePath[] { containerDir, containerSysDir })
{
NUnit.Framework.Assert.IsFalse(f.GetAbsolutePath() + " exists!!", f.Exists());
}
NUnit.Framework.Assert.IsFalse(targetFile.GetAbsolutePath() + " exists!!", targetFile
.Exists());
// Simulate RM sending an AppFinish event.
containerManager.Handle(new CMgrCompletedAppsEvent(Arrays.AsList(new ApplicationId
[] { appId }), CMgrCompletedAppsEvent.Reason.OnShutdown));
BaseContainerManagerTest.WaitForApplicationState(containerManager, cId.GetApplicationAttemptId
().GetApplicationId(), ApplicationState.Finished);
// Now ascertain that the resources are localised correctly.
foreach (FilePath f_1 in new FilePath[] { appDir, containerDir, appSysDir, containerSysDir })
{
// Wait for deletion. Deletion can happen long after AppFinish because of
// the async DeletionService
int timeout = 0;
while (f_1.Exists() && timeout++ < 15)
{
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsFalse(f_1.GetAbsolutePath() + " exists!!", f_1.Exists());
}
// Wait for deletion
int timeout_1 = 0;
while (targetFile.Exists() && timeout_1++ < 15)
{
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsFalse(targetFile.GetAbsolutePath() + " exists!!", targetFile
.Exists());
}
internal ResourceRetentionSet(DeletionService delService, long targetSize, IComparer
<LocalizedResource> cmp)
: this(delService, targetSize, new SortedDictionary <LocalizedResource, LocalResourcesTracker
>(cmp))
{
}
protected internal override LogHandler CreateLogHandler(Configuration conf, Context
context, DeletionService deletionService)
{
return(logHandler);
}
protected internal override ResourceLocalizationService CreateResourceLocalizationService
(ContainerExecutor exec, DeletionService deletionContext, Context context)
{
return(rsrcSrv);
}
public AppLogAggregatorImpl(Dispatcher dispatcher, DeletionService deletionService
, Configuration conf, ApplicationId appId, UserGroupInformation userUgi, NodeId
nodeId, LocalDirsHandlerService dirsHandler, Path remoteNodeLogFileForApp, ContainerLogsRetentionPolicy
retentionPolicy, IDictionary <ApplicationAccessType, string> appAcls, LogAggregationContext
logAggregationContext, Context context, FileContext lfs)
{
// This is temporary solution. The configuration will be deleted once
// we find a more scalable method to only write a single log file per LRS.
// This configuration is for debug and test purpose. By setting
// this configuration as true. We can break the lower bound of
// NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS.
// This variable is only for testing
this.dispatcher = dispatcher;
this.conf = conf;
this.delService = deletionService;
this.appId = appId;
this.applicationId = ConverterUtils.ToString(appId);
this.userUgi = userUgi;
this.dirsHandler = dirsHandler;
this.remoteNodeLogFileForApp = remoteNodeLogFileForApp;
this.remoteNodeTmpLogFileForApp = GetRemoteNodeTmpLogFileForApp();
this.retentionPolicy = retentionPolicy;
this.pendingContainers = new LinkedBlockingQueue <ContainerId>();
this.appAcls = appAcls;
this.lfs = lfs;
this.logAggregationContext = logAggregationContext;
this.context = context;
this.nodeId = nodeId;
int configuredRentionSize = conf.GetInt(NmLogAggregationNumLogFilesSizePerApp, DefaultNmLogAggregationNumLogFilesSizePerApp
);
if (configuredRentionSize <= 0)
{
this.retentionSize = DefaultNmLogAggregationNumLogFilesSizePerApp;
}
else
{
this.retentionSize = configuredRentionSize;
}
long configuredRollingMonitorInterval = conf.GetLong(YarnConfiguration.NmLogAggregationRollMonitoringIntervalSeconds
, YarnConfiguration.DefaultNmLogAggregationRollMonitoringIntervalSeconds);
bool debug_mode = conf.GetBoolean(NmLogAggregationDebugEnabled, DefaultNmLogAggregationDebugEnabled
);
if (configuredRollingMonitorInterval > 0 && configuredRollingMonitorInterval < NmLogAggregationMinRollMonitoringIntervalSeconds)
{
if (debug_mode)
{
this.rollingMonitorInterval = configuredRollingMonitorInterval;
}
else
{
Log.Warn("rollingMonitorIntervall should be more than or equal to " + NmLogAggregationMinRollMonitoringIntervalSeconds
+ " seconds. Using " + NmLogAggregationMinRollMonitoringIntervalSeconds + " seconds instead."
);
this.rollingMonitorInterval = NmLogAggregationMinRollMonitoringIntervalSeconds;
}
}
else
{
if (configuredRollingMonitorInterval <= 0)
{
Log.Warn("rollingMonitorInterval is set as " + configuredRollingMonitorInterval +
". " + "The log rolling mornitoring interval is disabled. " + "The logs will be aggregated after this application is finished."
);
}
else
{
Log.Warn("rollingMonitorInterval is set as " + configuredRollingMonitorInterval +
". " + "The logs will be aggregated every " + configuredRollingMonitorInterval
+ " seconds");
}
this.rollingMonitorInterval = configuredRollingMonitorInterval;
}
this.logAggregationInRolling = this.rollingMonitorInterval <= 0 || this.logAggregationContext
== null || this.logAggregationContext.GetRolledLogsIncludePattern() == null ||
this.logAggregationContext.GetRolledLogsIncludePattern().IsEmpty() ? false : true;
}
