public virtual ICollection <Path> DoContainerLogAggregation(AggregatedLogFormat.LogWriter
writer, bool appFinished)
{
AppLogAggregatorImpl.Log.Info("Uploading logs for container " + this.containerId
+ ". Current good log dirs are " + StringUtils.Join(",", this._enclosing.dirsHandler
.GetLogDirsForRead()));
AggregatedLogFormat.LogKey logKey = new AggregatedLogFormat.LogKey(this.containerId
);
AggregatedLogFormat.LogValue logValue = new AggregatedLogFormat.LogValue(this._enclosing
.dirsHandler.GetLogDirsForRead(), this.containerId, this._enclosing.userUgi.GetShortUserName
(), this._enclosing.logAggregationContext, this.uploadedFileMeta, appFinished);
try
{
writer.Append(logKey, logValue);
}
catch (Exception e)
{
AppLogAggregatorImpl.Log.Error("Couldn't upload logs for " + this.containerId + ". Skipping this container."
, e);
return(new HashSet <Path>());
}
Sharpen.Collections.AddAll(this.uploadedFileMeta, logValue.GetCurrentUpLoadedFileMeta
());
// if any of the previous uploaded logs have been deleted,
// we need to remove them from alreadyUploadedLogs
IEnumerable <string> mask = Iterables.Filter(this.uploadedFileMeta, new _Predicate_581
(logValue));
this.uploadedFileMeta = Sets.NewHashSet(mask);
return(logValue.GetCurrentUpLoadedFilesPath());
}
/// <exception cref="System.Exception"/>
private static void UploadEmptyContainerLogIntoRemoteDir(UserGroupInformation ugi
, Configuration configuration, IList <string> rootLogDirs, NodeId nodeId, ContainerId
containerId, Path appDir, FileSystem fs)
{
Path path = new Path(appDir, LogAggregationUtils.GetNodeString(nodeId) + Runtime.
CurrentTimeMillis());
AggregatedLogFormat.LogWriter writer = new AggregatedLogFormat.LogWriter(configuration
, path, ugi);
writer.WriteApplicationOwner(ugi.GetUserName());
IDictionary <ApplicationAccessType, string> appAcls = new Dictionary <ApplicationAccessType
, string>();
appAcls[ApplicationAccessType.ViewApp] = ugi.GetUserName();
writer.WriteApplicationACLs(appAcls);
DataOutputStream @out = writer.GetWriter().PrepareAppendKey(-1);
new AggregatedLogFormat.LogKey(containerId).Write(@out);
@out.Close();
@out = writer.GetWriter().PrepareAppendValue(-1);
new AggregatedLogFormat.LogValue(rootLogDirs, containerId, UserGroupInformation.GetCurrentUser
().GetShortUserName()).Write(@out, new HashSet <FilePath>());
@out.Close();
writer.Close();
}
private void UploadLogsForContainers(bool appFinished)
{
if (this.logAggregationDisabled)
{
return;
}
if (UserGroupInformation.IsSecurityEnabled())
{
Credentials systemCredentials = context.GetSystemCredentialsForApps()[appId];
if (systemCredentials != null)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Adding new framework-token for " + appId + " for log-aggregation: " +
systemCredentials.GetAllTokens() + "; userUgi=" + userUgi);
}
// this will replace old token
userUgi.AddCredentials(systemCredentials);
}
}
// Create a set of Containers whose logs will be uploaded in this cycle.
// It includes:
// a) all containers in pendingContainers: those containers are finished
// and satisfy the retentionPolicy.
// b) some set of running containers: For all the Running containers,
// we have ContainerLogsRetentionPolicy.AM_AND_FAILED_CONTAINERS_ONLY,
// so simply set wasContainerSuccessful as true to
// bypass FAILED_CONTAINERS check and find the running containers
// which satisfy the retentionPolicy.
ICollection <ContainerId> pendingContainerInThisCycle = new HashSet <ContainerId>();
this.pendingContainers.DrainTo(pendingContainerInThisCycle);
ICollection <ContainerId> finishedContainers = new HashSet <ContainerId>(pendingContainerInThisCycle
);
if (this.context.GetApplications()[this.appId] != null)
{
foreach (ContainerId container in this.context.GetApplications()[this.appId].GetContainers
().Keys)
{
if (ShouldUploadLogs(container, true))
{
pendingContainerInThisCycle.AddItem(container);
}
}
}
AggregatedLogFormat.LogWriter writer = null;
try
{
try
{
writer = new AggregatedLogFormat.LogWriter(this.conf, this.remoteNodeTmpLogFileForApp
, this.userUgi);
// Write ACLs once when the writer is created.
writer.WriteApplicationACLs(appAcls);
writer.WriteApplicationOwner(this.userUgi.GetShortUserName());
}
catch (IOException e1)
{
Log.Error("Cannot create writer for app " + this.applicationId + ". Skip log upload this time. "
, e1);
return;
}
bool uploadedLogsInThisCycle = false;
foreach (ContainerId container in pendingContainerInThisCycle)
{
AppLogAggregatorImpl.ContainerLogAggregator aggregator = null;
if (containerLogAggregators.Contains(container))
{
aggregator = containerLogAggregators[container];
}
else
{
aggregator = new AppLogAggregatorImpl.ContainerLogAggregator(this, container);
containerLogAggregators[container] = aggregator;
}
ICollection <Path> uploadedFilePathsInThisCycle = aggregator.DoContainerLogAggregation
(writer, appFinished);
if (uploadedFilePathsInThisCycle.Count > 0)
{
uploadedLogsInThisCycle = true;
this.delService.Delete(this.userUgi.GetShortUserName(), null, Sharpen.Collections.ToArray
(uploadedFilePathsInThisCycle, new Path[uploadedFilePathsInThisCycle.Count]));
}
// This container is finished, and all its logs have been uploaded,
// remove it from containerLogAggregators.
if (finishedContainers.Contains(container))
{
Sharpen.Collections.Remove(containerLogAggregators, container);
}
}
// Before upload logs, make sure the number of existing logs
// is smaller than the configured NM log aggregation retention size.
if (uploadedLogsInThisCycle)
{
CleanOldLogs();
}
if (writer != null)
{
writer.Close();
writer = null;
}
Path renamedPath = this.rollingMonitorInterval <= 0 ? remoteNodeLogFileForApp : new
Path(remoteNodeLogFileForApp.GetParent(), remoteNodeLogFileForApp.GetName() + "_"
+ Runtime.CurrentTimeMillis());
bool rename = uploadedLogsInThisCycle;
try
{
userUgi.DoAs(new _PrivilegedExceptionAction_304(this, rename, renamedPath));
}
catch (Exception e)
{
Log.Error("Failed to move temporary log file to final location: [" + remoteNodeTmpLogFileForApp
+ "] to [" + renamedPath + "]", e);
}
}
finally
{
if (writer != null)
{
writer.Close();
}
}
}
