/// <exception cref="System.Exception"/>
private static void UploadEmptyContainerLogIntoRemoteDir(UserGroupInformation ugi
, Configuration configuration, IList <string> rootLogDirs, NodeId nodeId, ContainerId
containerId, Path appDir, FileSystem fs)
{
Path path = new Path(appDir, LogAggregationUtils.GetNodeString(nodeId) + Runtime.
CurrentTimeMillis());
AggregatedLogFormat.LogWriter writer = new AggregatedLogFormat.LogWriter(configuration
, path, ugi);
writer.WriteApplicationOwner(ugi.GetUserName());
IDictionary <ApplicationAccessType, string> appAcls = new Dictionary <ApplicationAccessType
, string>();
appAcls[ApplicationAccessType.ViewApp] = ugi.GetUserName();
writer.WriteApplicationACLs(appAcls);
DataOutputStream @out = writer.GetWriter().PrepareAppendKey(-1);
new AggregatedLogFormat.LogKey(containerId).Write(@out);
@out.Close();
@out = writer.GetWriter().PrepareAppendValue(-1);
new AggregatedLogFormat.LogValue(rootLogDirs, containerId, UserGroupInformation.GetCurrentUser
().GetShortUserName()).Write(@out, new HashSet <FilePath>());
@out.Close();
writer.Close();
}
/// <exception cref="System.Exception"/>
public object Run()
{
try
{
FileSystem remoteFS = this._enclosing.GetFileSystem(this._enclosing.GetConfig());
Path appDir = LogAggregationUtils.GetRemoteAppLogDir(this._enclosing.remoteRootLogDir
, appId, user, this._enclosing.remoteRootLogDirSuffix);
appDir = appDir.MakeQualified(remoteFS.GetUri(), remoteFS.GetWorkingDirectory());
if (!this._enclosing.CheckExists(remoteFS, appDir, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Logaggregation.LogAggregationService
.AppDirPermissions))
{
Path suffixDir = LogAggregationUtils.GetRemoteLogSuffixedDir(this._enclosing.remoteRootLogDir
, user, this._enclosing.remoteRootLogDirSuffix);
suffixDir = suffixDir.MakeQualified(remoteFS.GetUri(), remoteFS.GetWorkingDirectory
());
if (!this._enclosing.CheckExists(remoteFS, suffixDir, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Logaggregation.LogAggregationService
.AppDirPermissions))
{
Path userDir = LogAggregationUtils.GetRemoteLogUserDir(this._enclosing.remoteRootLogDir
, user);
userDir = userDir.MakeQualified(remoteFS.GetUri(), remoteFS.GetWorkingDirectory()
);
if (!this._enclosing.CheckExists(remoteFS, userDir, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Logaggregation.LogAggregationService
.AppDirPermissions))
{
this._enclosing.CreateDir(remoteFS, userDir, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Logaggregation.LogAggregationService
.AppDirPermissions);
}
this._enclosing.CreateDir(remoteFS, suffixDir, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Logaggregation.LogAggregationService
.AppDirPermissions);
}
this._enclosing.CreateDir(remoteFS, appDir, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Logaggregation.LogAggregationService
.AppDirPermissions);
}
}
catch (IOException e)
{
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Logaggregation.LogAggregationService
.Log.Error("Failed to setup application log directory for " + appId, e);
throw;
}
return(null);
}
protected internal override void Render(HtmlBlock.Block html)
{
ContainerId containerId = VerifyAndGetContainerId(html);
NodeId nodeId = VerifyAndGetNodeId(html);
string appOwner = VerifyAndGetAppOwner(html);
AggregatedLogsBlock.LogLimits logLimits = VerifyAndGetLogLimits(html);
if (containerId == null || nodeId == null || appOwner == null || appOwner.IsEmpty
() || logLimits == null)
{
return;
}
ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId
();
string logEntity = $(YarnWebParams.EntityString);
if (logEntity == null || logEntity.IsEmpty())
{
logEntity = containerId.ToString();
}
if (!conf.GetBoolean(YarnConfiguration.LogAggregationEnabled, YarnConfiguration.DefaultLogAggregationEnabled
))
{
html.H1().("Aggregation is not enabled. Try the nodemanager at " + nodeId).();
return;
}
Path remoteRootLogDir = new Path(conf.Get(YarnConfiguration.NmRemoteAppLogDir, YarnConfiguration
.DefaultNmRemoteAppLogDir));
Path remoteAppDir = LogAggregationUtils.GetRemoteAppLogDir(remoteRootLogDir, applicationId
, appOwner, LogAggregationUtils.GetRemoteNodeLogDirSuffix(conf));
RemoteIterator <FileStatus> nodeFiles;
try
{
Path qualifiedLogDir = FileContext.GetFileContext(conf).MakeQualified(remoteAppDir
);
nodeFiles = FileContext.GetFileContext(qualifiedLogDir.ToUri(), conf).ListStatus(
remoteAppDir);
}
catch (FileNotFoundException)
{
html.H1().("Logs not available for " + logEntity + ". Aggregation may not be complete, "
+ "Check back later or try the nodemanager at " + nodeId).();
return;
}
catch (Exception)
{
html.H1().("Error getting logs at " + nodeId).();
return;
}
bool foundLog = false;
string desiredLogType = $(YarnWebParams.ContainerLogType);
try
{
while (nodeFiles.HasNext())
{
AggregatedLogFormat.LogReader reader = null;
try
{
FileStatus thisNodeFile = nodeFiles.Next();
if (!thisNodeFile.GetPath().GetName().Contains(LogAggregationUtils.GetNodeString(
nodeId)) || thisNodeFile.GetPath().GetName().EndsWith(LogAggregationUtils.TmpFileSuffix
))
{
continue;
}
long logUploadedTime = thisNodeFile.GetModificationTime();
reader = new AggregatedLogFormat.LogReader(conf, thisNodeFile.GetPath());
string owner = null;
IDictionary <ApplicationAccessType, string> appAcls = null;
try
{
owner = reader.GetApplicationOwner();
appAcls = reader.GetApplicationAcls();
}
catch (IOException e)
{
Log.Error("Error getting logs for " + logEntity, e);
continue;
}
ApplicationACLsManager aclsManager = new ApplicationACLsManager(conf);
aclsManager.AddApplication(applicationId, appAcls);
string remoteUser = Request().GetRemoteUser();
UserGroupInformation callerUGI = null;
if (remoteUser != null)
{
callerUGI = UserGroupInformation.CreateRemoteUser(remoteUser);
}
if (callerUGI != null && !aclsManager.CheckAccess(callerUGI, ApplicationAccessType
.ViewApp, owner, applicationId))
{
html.H1().("User [" + remoteUser + "] is not authorized to view the logs for " +
logEntity + " in log file [" + thisNodeFile.GetPath().GetName() + "]").();
Log.Error("User [" + remoteUser + "] is not authorized to view the logs for " + logEntity
);
continue;
}
AggregatedLogFormat.ContainerLogsReader logReader = reader.GetContainerLogsReader
(containerId);
if (logReader == null)
{
continue;
}
foundLog = ReadContainerLogs(html, logReader, logLimits, desiredLogType, logUploadedTime
);
}
catch (IOException ex)
{
Log.Error("Error getting logs for " + logEntity, ex);
continue;
}
finally
{
if (reader != null)
{
reader.Close();
}
}
}
if (!foundLog)
{
if (desiredLogType.IsEmpty())
{
html.H1("No logs available for container " + containerId.ToString());
}
else
{
html.H1("Unable to locate '" + desiredLogType + "' log for container " + containerId
.ToString());
}
}
}
catch (IOException e)
{
html.H1().("Error getting logs for " + logEntity).();
Log.Error("Error getting logs for " + logEntity, e);
}
}
public bool Apply(FileStatus next)
{
return(next.GetPath().GetName().Contains(LogAggregationUtils.GetNodeString(this._enclosing
.nodeId)) && !next.GetPath().GetName().EndsWith(LogAggregationUtils.TmpFileSuffix
));
}
internal virtual Path GetRemoteAppLogDir(ApplicationId appId, string user)
{
return(LogAggregationUtils.GetRemoteAppLogDir(this.remoteRootLogDir, appId, user,
this.remoteRootLogDirSuffix));
}
internal virtual Path GetRemoteNodeLogFileForApp(ApplicationId appId, string user
)
{
return(LogAggregationUtils.GetRemoteNodeLogFileForApp(this.remoteRootLogDir, appId
, user, this.nodeId, this.remoteRootLogDirSuffix));
}
