protected internal virtual void InitAppAggregator(ApplicationId appId, string user
, Credentials credentials, ContainerLogsRetentionPolicy logRetentionPolicy, IDictionary
<ApplicationAccessType, string> appAcls, LogAggregationContext logAggregationContext
)
{
// Get user's FileSystem credentials
UserGroupInformation userUgi = UserGroupInformation.CreateRemoteUser(user);
if (credentials != null)
{
userUgi.AddCredentials(credentials);
}
// New application
AppLogAggregator appLogAggregator = new AppLogAggregatorImpl(this.dispatcher, this
.deletionService, GetConfig(), appId, userUgi, this.nodeId, dirsHandler, GetRemoteNodeLogFileForApp
(appId, user), logRetentionPolicy, appAcls, logAggregationContext, this.context,
GetLocalFileContext(GetConfig()));
if (this.appLogAggregators.PutIfAbsent(appId, appLogAggregator) != null)
{
throw new YarnRuntimeException("Duplicate initApp for " + appId);
}
// wait until check for existing aggregator to create dirs
YarnRuntimeException appDirException = null;
try
{
// Create the app dir
CreateAppDir(user, appId, userUgi);
}
catch (Exception e)
{
appLogAggregator.DisableLogAggregation();
if (!(e is YarnRuntimeException))
{
appDirException = new YarnRuntimeException(e);
}
else
{
appDirException = (YarnRuntimeException)e;
}
}
// TODO Get the user configuration for the list of containers that need log
// aggregation.
// Schedule the aggregator.
Runnable aggregatorWrapper = new _Runnable_381(this, appLogAggregator, appId, userUgi
);
this.threadPool.Execute(aggregatorWrapper);
if (appDirException != null)
{
throw appDirException;
}
}
public LogHandlerAppStartedEvent(ApplicationId appId, string user, Credentials credentials
, ContainerLogsRetentionPolicy retentionPolicy, IDictionary <ApplicationAccessType
, string> appAcls, LogAggregationContext logAggregationContext)
: base(LogHandlerEventType.ApplicationStarted)
{
this.applicationId = appId;
this.user = user;
this.credentials = credentials;
this.retentionPolicy = retentionPolicy;
this.appAcls = appAcls;
this.logAggregationContext = logAggregationContext;
}
public AppLogAggregatorImpl(Dispatcher dispatcher, DeletionService deletionService
, Configuration conf, ApplicationId appId, UserGroupInformation userUgi, NodeId
nodeId, LocalDirsHandlerService dirsHandler, Path remoteNodeLogFileForApp, ContainerLogsRetentionPolicy
retentionPolicy, IDictionary <ApplicationAccessType, string> appAcls, LogAggregationContext
logAggregationContext, Context context, FileContext lfs)
{
// This is temporary solution. The configuration will be deleted once
// we find a more scalable method to only write a single log file per LRS.
// This configuration is for debug and test purpose. By setting
// this configuration as true. We can break the lower bound of
// NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS.
// This variable is only for testing
this.dispatcher = dispatcher;
this.conf = conf;
this.delService = deletionService;
this.appId = appId;
this.applicationId = ConverterUtils.ToString(appId);
this.userUgi = userUgi;
this.dirsHandler = dirsHandler;
this.remoteNodeLogFileForApp = remoteNodeLogFileForApp;
this.remoteNodeTmpLogFileForApp = GetRemoteNodeTmpLogFileForApp();
this.retentionPolicy = retentionPolicy;
this.pendingContainers = new LinkedBlockingQueue <ContainerId>();
this.appAcls = appAcls;
this.lfs = lfs;
this.logAggregationContext = logAggregationContext;
this.context = context;
this.nodeId = nodeId;
int configuredRentionSize = conf.GetInt(NmLogAggregationNumLogFilesSizePerApp, DefaultNmLogAggregationNumLogFilesSizePerApp
);
if (configuredRentionSize <= 0)
{
this.retentionSize = DefaultNmLogAggregationNumLogFilesSizePerApp;
}
else
{
this.retentionSize = configuredRentionSize;
}
long configuredRollingMonitorInterval = conf.GetLong(YarnConfiguration.NmLogAggregationRollMonitoringIntervalSeconds
, YarnConfiguration.DefaultNmLogAggregationRollMonitoringIntervalSeconds);
bool debug_mode = conf.GetBoolean(NmLogAggregationDebugEnabled, DefaultNmLogAggregationDebugEnabled
);
if (configuredRollingMonitorInterval > 0 && configuredRollingMonitorInterval < NmLogAggregationMinRollMonitoringIntervalSeconds)
{
if (debug_mode)
{
this.rollingMonitorInterval = configuredRollingMonitorInterval;
}
else
{
Log.Warn("rollingMonitorIntervall should be more than or equal to " + NmLogAggregationMinRollMonitoringIntervalSeconds
+ " seconds. Using " + NmLogAggregationMinRollMonitoringIntervalSeconds + " seconds instead."
);
this.rollingMonitorInterval = NmLogAggregationMinRollMonitoringIntervalSeconds;
}
}
else
{
if (configuredRollingMonitorInterval <= 0)
{
Log.Warn("rollingMonitorInterval is set as " + configuredRollingMonitorInterval +
". " + "The log rolling mornitoring interval is disabled. " + "The logs will be aggregated after this application is finished."
);
}
else
{
Log.Warn("rollingMonitorInterval is set as " + configuredRollingMonitorInterval +
". " + "The logs will be aggregated every " + configuredRollingMonitorInterval
+ " seconds");
}
this.rollingMonitorInterval = configuredRollingMonitorInterval;
}
this.logAggregationInRolling = this.rollingMonitorInterval <= 0 || this.logAggregationContext
== null || this.logAggregationContext.GetRolledLogsIncludePattern() == null ||
this.logAggregationContext.GetRolledLogsIncludePattern().IsEmpty() ? false : true;
}
public LogHandlerAppStartedEvent(ApplicationId appId, string user, Credentials credentials
, ContainerLogsRetentionPolicy retentionPolicy, IDictionary <ApplicationAccessType
, string> appAcls)
: this(appId, user, credentials, retentionPolicy, appAcls, null)
{
}
private void InitApp(ApplicationId appId, string user, Credentials credentials, ContainerLogsRetentionPolicy
logRetentionPolicy, IDictionary <ApplicationAccessType, string> appAcls, LogAggregationContext
logAggregationContext)
{
ApplicationEvent eventResponse;
try
{
VerifyAndCreateRemoteLogDir(GetConfig());
InitAppAggregator(appId, user, credentials, logRetentionPolicy, appAcls, logAggregationContext
);
eventResponse = new ApplicationEvent(appId, ApplicationEventType.ApplicationLogHandlingInited
);
}
catch (YarnRuntimeException e)
{
Log.Warn("Application failed to init aggregation", e);
eventResponse = new ApplicationEvent(appId, ApplicationEventType.ApplicationLogHandlingFailed
);
}
this.dispatcher.GetEventHandler().Handle(eventResponse);
}
