public ContainerTokenIdentifier(ContainerId containerID, string hostName, string
appSubmitter, Resource r, long expiryTimeStamp, int masterKeyId, long rmIdentifier
, Priority priority, long creationTime, LogAggregationContext logAggregationContext
)
{
YarnSecurityTokenProtos.ContainerTokenIdentifierProto.Builder builder = YarnSecurityTokenProtos.ContainerTokenIdentifierProto
.NewBuilder();
if (containerID != null)
{
builder.SetContainerId(((ContainerIdPBImpl)containerID).GetProto());
}
builder.SetNmHostAddr(hostName);
builder.SetAppSubmitter(appSubmitter);
if (r != null)
{
builder.SetResource(((ResourcePBImpl)r).GetProto());
}
builder.SetExpiryTimeStamp(expiryTimeStamp);
builder.SetMasterKeyId(masterKeyId);
builder.SetRmIdentifier(rmIdentifier);
if (priority != null)
{
builder.SetPriority(((PriorityPBImpl)priority).GetProto());
}
builder.SetCreationTime(creationTime);
if (logAggregationContext != null)
{
builder.SetLogAggregationContext(((LogAggregationContextPBImpl)logAggregationContext
).GetProto());
}
proto = ((YarnSecurityTokenProtos.ContainerTokenIdentifierProto)builder.Build());
}
public ApplicationInitEvent(ApplicationId appId, IDictionary <ApplicationAccessType
, string> acls, LogAggregationContext logAggregationContext)
: base(appId, ApplicationEventType.InitApplication)
{
this.applicationACLs = acls;
this.logAggregationContext = logAggregationContext;
}
public SchedulerApplicationAttempt(ApplicationAttemptId applicationAttemptId, string
user, Queue queue, ActiveUsersManager activeUsersManager, RMContext rmContext)
{
// This pendingRelease is used in work-preserving recovery scenario to keep
// track of the AM's outstanding release requests. RM on recovery could
// receive the release request form AM before it receives the container status
// from NM for recovery. In this case, the to-be-recovered containers reported
// by NM should not be recovered.
// Time of the last container scheduled at the current allowed level
Preconditions.CheckNotNull(rmContext, "RMContext should not be null");
this.rmContext = rmContext;
this.appSchedulingInfo = new AppSchedulingInfo(applicationAttemptId, user, queue,
activeUsersManager, rmContext.GetEpoch());
this.queue = queue;
this.pendingRelease = new HashSet <ContainerId>();
this.attemptId = applicationAttemptId;
if (rmContext.GetRMApps() != null && rmContext.GetRMApps().Contains(applicationAttemptId
.GetApplicationId()))
{
ApplicationSubmissionContext appSubmissionContext = rmContext.GetRMApps()[applicationAttemptId
.GetApplicationId()].GetApplicationSubmissionContext();
if (appSubmissionContext != null)
{
unmanagedAM = appSubmissionContext.GetUnmanagedAM();
this.logAggregationContext = appSubmissionContext.GetLogAggregationContext();
}
}
}
public virtual void TestLogAggregationContextPassedIntoContainerToken()
{
MockRM rm1 = new MockRM(conf);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("127.0.0.1:1234", 8000);
MockNM nm2 = rm1.RegisterNode("127.0.0.1:2345", 8000);
// LogAggregationContext is set as null
NUnit.Framework.Assert.IsNull(GetLogAggregationContextFromContainerToken(rm1, nm1
, null));
// create a not-null LogAggregationContext
LogAggregationContext logAggregationContext = LogAggregationContext.NewInstance("includePattern"
, "excludePattern", "rolledLogsIncludePattern", "rolledLogsExcludePattern");
LogAggregationContext returned = GetLogAggregationContextFromContainerToken(rm1,
nm2, logAggregationContext);
NUnit.Framework.Assert.AreEqual("includePattern", returned.GetIncludePattern());
NUnit.Framework.Assert.AreEqual("excludePattern", returned.GetExcludePattern());
NUnit.Framework.Assert.AreEqual("rolledLogsIncludePattern", returned.GetRolledLogsIncludePattern
());
NUnit.Framework.Assert.AreEqual("rolledLogsExcludePattern", returned.GetRolledLogsExcludePattern
());
rm1.Stop();
}
public override Token CreateContainerToken(ContainerId containerId, NodeId nodeId
, string appSubmitter, Resource capability, Priority priority, long createTime,
LogAggregationContext logAggregationContext)
{
this._enclosing._enclosing.numRetries++;
return(base.CreateContainerToken(containerId, nodeId, appSubmitter, capability, priority
, createTime, logAggregationContext));
}
/// <exception cref="System.Exception"/>
public virtual RMApp SubmitApp(int masterMemory, LogAggregationContext logAggregationContext
)
{
return(SubmitApp(masterMemory, string.Empty, UserGroupInformation.GetCurrentUser(
).GetShortUserName(), null, false, null, base.GetConfig().GetInt(YarnConfiguration
.RmAmMaxAttempts, YarnConfiguration.DefaultRmAmMaxAttempts), null, null, true, false
, false, null, 0, logAggregationContext, true));
}
public override void SetLogAggregationContext(LogAggregationContext logAggregationContext
)
{
MaybeInitBuilder();
if (logAggregationContext == null)
{
builder.ClearLogAggregationContext();
}
this.logAggregationContext = logAggregationContext;
}
public LogHandlerAppStartedEvent(ApplicationId appId, string user, Credentials credentials
, ContainerLogsRetentionPolicy retentionPolicy, IDictionary <ApplicationAccessType
, string> appAcls, LogAggregationContext logAggregationContext)
: base(LogHandlerEventType.ApplicationStarted)
{
this.applicationId = appId;
this.user = user;
this.credentials = credentials;
this.retentionPolicy = retentionPolicy;
this.appAcls = appAcls;
this.logAggregationContext = logAggregationContext;
}
/// <exception cref="System.IO.IOException"/>
public static Token CreateContainerToken(ContainerId cId, long rmIdentifier, NodeId
nodeId, string user, NMContainerTokenSecretManager containerTokenSecretManager,
LogAggregationContext logAggregationContext)
{
Resource r = BuilderUtils.NewResource(1024, 1);
ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
cId, nodeId.ToString(), user, r, Runtime.CurrentTimeMillis() + 100000L, 123, rmIdentifier
, Priority.NewInstance(0), 0, logAggregationContext);
Token containerToken = BuilderUtils.NewContainerToken(nodeId, containerTokenSecretManager
.RetrievePassword(containerTokenIdentifier), containerTokenIdentifier);
return(containerToken);
}
public override LogAggregationContext GetLogAggregationContext()
{
YarnProtos.ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
if (this.logAggregationContext != null)
{
return(this.logAggregationContext);
}
// Else via proto
if (!p.HasLogAggregationContext())
{
return(null);
}
logAggregationContext = ConvertFromProtoFormat(p.GetLogAggregationContext());
return(logAggregationContext);
}
public LogValue(IList <string> rootLogDirs, ContainerId containerId, string user,
LogAggregationContext logAggregationContext, ICollection <string> alreadyUploadedLogFiles
, bool appFinished)
{
// TODO Maybe add a version string here. Instead of changing the version of
// the entire k-v format
this.rootLogDirs = new AList <string>(rootLogDirs);
this.containerId = containerId;
this.user = user;
// Ensure logs are processed in lexical order
this.rootLogDirs.Sort();
this.logAggregationContext = logAggregationContext;
this.alreadyUploadedLogFiles = alreadyUploadedLogFiles;
this.appFinished = appFinished;
}
/// <exception cref="System.Exception"/>
private StartContainersResponse StartContainer(Context context, ContainerManagerImpl
cm, ContainerId cid, ContainerLaunchContext clc, LogAggregationContext logAggregationContext
)
{
UserGroupInformation user = UserGroupInformation.CreateRemoteUser(cid.GetApplicationAttemptId
().ToString());
StartContainerRequest scReq = StartContainerRequest.NewInstance(clc, TestContainerManager
.CreateContainerToken(cid, 0, context.GetNodeId(), user.GetShortUserName(), context
.GetContainerTokenSecretManager(), logAggregationContext));
IList <StartContainerRequest> scReqList = new AList <StartContainerRequest>();
scReqList.AddItem(scReq);
NMTokenIdentifier nmToken = new NMTokenIdentifier(cid.GetApplicationAttemptId(),
context.GetNodeId(), user.GetShortUserName(), context.GetNMTokenSecretManager().
GetCurrentKey().GetKeyId());
user.AddTokenIdentifier(nmToken);
return(user.DoAs(new _PrivilegedExceptionAction_264(cm, scReqList)));
}
public virtual void TestApplicationRecovery()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.NmRecoveryEnabled, true);
conf.Set(YarnConfiguration.NmAddress, "localhost:1234");
conf.SetBoolean(YarnConfiguration.YarnAclEnable, true);
conf.Set(YarnConfiguration.YarnAdminAcl, "yarn_admin_user");
NMStateStoreService stateStore = new NMMemoryStateStoreService();
stateStore.Init(conf);
stateStore.Start();
Context context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf
), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore
);
ContainerManagerImpl cm = CreateContainerManager(context);
cm.Init(conf);
cm.Start();
// simulate registration with RM
MasterKey masterKey = new MasterKeyPBImpl();
masterKey.SetKeyId(123);
masterKey.SetBytes(ByteBuffer.Wrap(new byte[] { 123 }));
context.GetContainerTokenSecretManager().SetMasterKey(masterKey);
context.GetNMTokenSecretManager().SetMasterKey(masterKey);
// add an application by starting a container
string appUser = "******";
string modUser = "******";
string viewUser = "******";
string enemyUser = "******";
ApplicationId appId = ApplicationId.NewInstance(0, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
ContainerId cid = ContainerId.NewContainerId(attemptId, 1);
IDictionary <string, LocalResource> localResources = Collections.EmptyMap();
IDictionary <string, string> containerEnv = Sharpen.Collections.EmptyMap();
IList <string> containerCmds = Sharpen.Collections.EmptyList();
IDictionary <string, ByteBuffer> serviceData = Sharpen.Collections.EmptyMap();
Credentials containerCreds = new Credentials();
DataOutputBuffer dob = new DataOutputBuffer();
containerCreds.WriteTokenStorageToStream(dob);
ByteBuffer containerTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>();
acls[ApplicationAccessType.ModifyApp] = modUser;
acls[ApplicationAccessType.ViewApp] = viewUser;
ContainerLaunchContext clc = ContainerLaunchContext.NewInstance(localResources, containerEnv
, containerCmds, serviceData, containerTokens, acls);
// create the logAggregationContext
LogAggregationContext logAggregationContext = LogAggregationContext.NewInstance("includePattern"
, "excludePattern", "includePatternInRollingAggregation", "excludePatternInRollingAggregation"
);
StartContainersResponse startResponse = StartContainer(context, cm, cid, clc, logAggregationContext
);
NUnit.Framework.Assert.IsTrue(startResponse.GetFailedRequests().IsEmpty());
NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = context.GetApplications()[appId];
NUnit.Framework.Assert.IsNotNull(app);
WaitForAppState(app, ApplicationState.Initing);
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId));
// reset container manager and verify app recovered with proper acls
cm.Stop();
context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new
NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore);
cm = CreateContainerManager(context);
cm.Init(conf);
cm.Start();
NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count);
app = context.GetApplications()[appId];
NUnit.Framework.Assert.IsNotNull(app);
// check whether LogAggregationContext is recovered correctly
LogAggregationContext recovered = ((ApplicationImpl)app).GetLogAggregationContext
();
NUnit.Framework.Assert.IsNotNull(recovered);
NUnit.Framework.Assert.AreEqual(logAggregationContext.GetIncludePattern(), recovered
.GetIncludePattern());
NUnit.Framework.Assert.AreEqual(logAggregationContext.GetExcludePattern(), recovered
.GetExcludePattern());
NUnit.Framework.Assert.AreEqual(logAggregationContext.GetRolledLogsIncludePattern
(), recovered.GetRolledLogsIncludePattern());
NUnit.Framework.Assert.AreEqual(logAggregationContext.GetRolledLogsExcludePattern
(), recovered.GetRolledLogsExcludePattern());
WaitForAppState(app, ApplicationState.Initing);
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId));
// simulate application completion
IList <ApplicationId> finishedApps = new AList <ApplicationId>();
finishedApps.AddItem(appId);
cm.Handle(new CMgrCompletedAppsEvent(finishedApps, CMgrCompletedAppsEvent.Reason.
ByResourcemanager));
WaitForAppState(app, ApplicationState.ApplicationResourcesCleaningup);
// restart and verify app is marked for finishing
cm.Stop();
context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new
NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore);
cm = CreateContainerManager(context);
cm.Init(conf);
cm.Start();
NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count);
app = context.GetApplications()[appId];
NUnit.Framework.Assert.IsNotNull(app);
WaitForAppState(app, ApplicationState.ApplicationResourcesCleaningup);
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId));
// simulate log aggregation completion
app.Handle(new ApplicationEvent(app.GetAppId(), ApplicationEventType.ApplicationResourcesCleanedup
));
NUnit.Framework.Assert.AreEqual(app.GetApplicationState(), ApplicationState.Finished
);
app.Handle(new ApplicationEvent(app.GetAppId(), ApplicationEventType.ApplicationLogHandlingFinished
));
// restart and verify app is no longer present after recovery
cm.Stop();
context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new
NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore);
cm = CreateContainerManager(context);
cm.Init(conf);
cm.Start();
NUnit.Framework.Assert.IsTrue(context.GetApplications().IsEmpty());
cm.Stop();
}
private YarnProtos.LogAggregationContextProto ConvertToProtoFormat(LogAggregationContext
t)
{
return(((LogAggregationContextPBImpl)t).GetProto());
}
/// <exception cref="System.Exception"/>
public virtual RMApp SubmitApp(int masterMemory, string name, string user, IDictionary
<ApplicationAccessType, string> acls, bool unmanaged, string queue, int maxAppAttempts
, Credentials ts, string appType, bool waitForAccepted, bool keepContainers, bool
isAppIdProvided, ApplicationId applicationId, long attemptFailuresValidityInterval
, LogAggregationContext logAggregationContext, bool cancelTokensWhenComplete)
{
ApplicationId appId = isAppIdProvided ? applicationId : null;
ApplicationClientProtocol client = GetClientRMService();
if (!isAppIdProvided)
{
GetNewApplicationResponse resp = client.GetNewApplication(Org.Apache.Hadoop.Yarn.Util.Records
.NewRecord <GetNewApplicationRequest>());
appId = resp.GetApplicationId();
}
SubmitApplicationRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <SubmitApplicationRequest
>();
ApplicationSubmissionContext sub = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <
ApplicationSubmissionContext>();
sub.SetKeepContainersAcrossApplicationAttempts(keepContainers);
sub.SetApplicationId(appId);
sub.SetApplicationName(name);
sub.SetMaxAppAttempts(maxAppAttempts);
if (unmanaged)
{
sub.SetUnmanagedAM(true);
}
if (queue != null)
{
sub.SetQueue(queue);
}
sub.SetApplicationType(appType);
ContainerLaunchContext clc = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext
>();
Resource capability = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Resource>();
capability.SetMemory(masterMemory);
sub.SetResource(capability);
clc.SetApplicationACLs(acls);
if (ts != null && UserGroupInformation.IsSecurityEnabled())
{
DataOutputBuffer dob = new DataOutputBuffer();
ts.WriteTokenStorageToStream(dob);
ByteBuffer securityTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
clc.SetTokens(securityTokens);
}
sub.SetAMContainerSpec(clc);
sub.SetAttemptFailuresValidityInterval(attemptFailuresValidityInterval);
if (logAggregationContext != null)
{
sub.SetLogAggregationContext(logAggregationContext);
}
sub.SetCancelTokensWhenComplete(cancelTokensWhenComplete);
req.SetApplicationSubmissionContext(sub);
UserGroupInformation fakeUser = UserGroupInformation.CreateUserForTesting(user, new
string[] { "someGroup" });
PrivilegedAction <SubmitApplicationResponse> action = new _PrivilegedAction_415().
SetClientReq(client, req);
fakeUser.DoAs(action);
// make sure app is immediately available after submit
if (waitForAccepted)
{
WaitForState(appId, RMAppState.Accepted);
}
RMApp rmApp = GetRMContext().GetRMApps()[appId];
// unmanaged AM won't go to RMAppAttemptState.SCHEDULED.
if (waitForAccepted && !unmanaged)
{
WaitForState(rmApp.GetCurrentAppAttempt().GetAppAttemptId(), RMAppAttemptState.Scheduled
);
}
return(rmApp);
}
private void InitApp(ApplicationId appId, string user, Credentials credentials, ContainerLogsRetentionPolicy
logRetentionPolicy, IDictionary <ApplicationAccessType, string> appAcls, LogAggregationContext
logAggregationContext)
{
ApplicationEvent eventResponse;
try
{
VerifyAndCreateRemoteLogDir(GetConfig());
InitAppAggregator(appId, user, credentials, logRetentionPolicy, appAcls, logAggregationContext
);
eventResponse = new ApplicationEvent(appId, ApplicationEventType.ApplicationLogHandlingInited
);
}
catch (YarnRuntimeException e)
{
Log.Warn("Application failed to init aggregation", e);
eventResponse = new ApplicationEvent(appId, ApplicationEventType.ApplicationLogHandlingFailed
);
}
this.dispatcher.GetEventHandler().Handle(eventResponse);
}
protected internal virtual void InitAppAggregator(ApplicationId appId, string user
, Credentials credentials, ContainerLogsRetentionPolicy logRetentionPolicy, IDictionary
<ApplicationAccessType, string> appAcls, LogAggregationContext logAggregationContext
)
{
// Get user's FileSystem credentials
UserGroupInformation userUgi = UserGroupInformation.CreateRemoteUser(user);
if (credentials != null)
{
userUgi.AddCredentials(credentials);
}
// New application
AppLogAggregator appLogAggregator = new AppLogAggregatorImpl(this.dispatcher, this
.deletionService, GetConfig(), appId, userUgi, this.nodeId, dirsHandler, GetRemoteNodeLogFileForApp
(appId, user), logRetentionPolicy, appAcls, logAggregationContext, this.context,
GetLocalFileContext(GetConfig()));
if (this.appLogAggregators.PutIfAbsent(appId, appLogAggregator) != null)
{
throw new YarnRuntimeException("Duplicate initApp for " + appId);
}
// wait until check for existing aggregator to create dirs
YarnRuntimeException appDirException = null;
try
{
// Create the app dir
CreateAppDir(user, appId, userUgi);
}
catch (Exception e)
{
appLogAggregator.DisableLogAggregation();
if (!(e is YarnRuntimeException))
{
appDirException = new YarnRuntimeException(e);
}
else
{
appDirException = (YarnRuntimeException)e;
}
}
// TODO Get the user configuration for the list of containers that need log
// aggregation.
// Schedule the aggregator.
Runnable aggregatorWrapper = new _Runnable_381(this, appLogAggregator, appId, userUgi
);
this.threadPool.Execute(aggregatorWrapper);
if (appDirException != null)
{
throw appDirException;
}
}
public AppLogAggregatorImpl(Dispatcher dispatcher, DeletionService deletionService
, Configuration conf, ApplicationId appId, UserGroupInformation userUgi, NodeId
nodeId, LocalDirsHandlerService dirsHandler, Path remoteNodeLogFileForApp, ContainerLogsRetentionPolicy
retentionPolicy, IDictionary <ApplicationAccessType, string> appAcls, LogAggregationContext
logAggregationContext, Context context, FileContext lfs)
{
// This is temporary solution. The configuration will be deleted once
// we find a more scalable method to only write a single log file per LRS.
// This configuration is for debug and test purpose. By setting
// this configuration as true. We can break the lower bound of
// NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS.
// This variable is only for testing
this.dispatcher = dispatcher;
this.conf = conf;
this.delService = deletionService;
this.appId = appId;
this.applicationId = ConverterUtils.ToString(appId);
this.userUgi = userUgi;
this.dirsHandler = dirsHandler;
this.remoteNodeLogFileForApp = remoteNodeLogFileForApp;
this.remoteNodeTmpLogFileForApp = GetRemoteNodeTmpLogFileForApp();
this.retentionPolicy = retentionPolicy;
this.pendingContainers = new LinkedBlockingQueue <ContainerId>();
this.appAcls = appAcls;
this.lfs = lfs;
this.logAggregationContext = logAggregationContext;
this.context = context;
this.nodeId = nodeId;
int configuredRentionSize = conf.GetInt(NmLogAggregationNumLogFilesSizePerApp, DefaultNmLogAggregationNumLogFilesSizePerApp
);
if (configuredRentionSize <= 0)
{
this.retentionSize = DefaultNmLogAggregationNumLogFilesSizePerApp;
}
else
{
this.retentionSize = configuredRentionSize;
}
long configuredRollingMonitorInterval = conf.GetLong(YarnConfiguration.NmLogAggregationRollMonitoringIntervalSeconds
, YarnConfiguration.DefaultNmLogAggregationRollMonitoringIntervalSeconds);
bool debug_mode = conf.GetBoolean(NmLogAggregationDebugEnabled, DefaultNmLogAggregationDebugEnabled
);
if (configuredRollingMonitorInterval > 0 && configuredRollingMonitorInterval < NmLogAggregationMinRollMonitoringIntervalSeconds)
{
if (debug_mode)
{
this.rollingMonitorInterval = configuredRollingMonitorInterval;
}
else
{
Log.Warn("rollingMonitorIntervall should be more than or equal to " + NmLogAggregationMinRollMonitoringIntervalSeconds
+ " seconds. Using " + NmLogAggregationMinRollMonitoringIntervalSeconds + " seconds instead."
);
this.rollingMonitorInterval = NmLogAggregationMinRollMonitoringIntervalSeconds;
}
}
else
{
if (configuredRollingMonitorInterval <= 0)
{
Log.Warn("rollingMonitorInterval is set as " + configuredRollingMonitorInterval +
". " + "The log rolling mornitoring interval is disabled. " + "The logs will be aggregated after this application is finished."
);
}
else
{
Log.Warn("rollingMonitorInterval is set as " + configuredRollingMonitorInterval +
". " + "The logs will be aggregated every " + configuredRollingMonitorInterval
+ " seconds");
}
this.rollingMonitorInterval = configuredRollingMonitorInterval;
}
this.logAggregationInRolling = this.rollingMonitorInterval <= 0 || this.logAggregationContext
== null || this.logAggregationContext.GetRolledLogsIncludePattern() == null ||
this.logAggregationContext.GetRolledLogsIncludePattern().IsEmpty() ? false : true;
}
/// <summary>Helper function for creating ContainerTokens</summary>
/// <param name="containerId"/>
/// <param name="nodeId"/>
/// <param name="appSubmitter"/>
/// <param name="capability"/>
/// <param name="priority"/>
/// <param name="createTime"/>
/// <param name="logAggregationContext"/>
/// <returns>the container-token</returns>
public virtual Token CreateContainerToken(ContainerId containerId, NodeId nodeId,
string appSubmitter, Resource capability, Priority priority, long createTime, LogAggregationContext
logAggregationContext)
{
byte[] password;
ContainerTokenIdentifier tokenIdentifier;
long expiryTimeStamp = Runtime.CurrentTimeMillis() + containerTokenExpiryInterval;
// Lock so that we use the same MasterKey's keyId and its bytes
this.readLock.Lock();
try
{
tokenIdentifier = new ContainerTokenIdentifier(containerId, nodeId.ToString(), appSubmitter
, capability, expiryTimeStamp, this.currentMasterKey.GetMasterKey().GetKeyId(),
ResourceManager.GetClusterTimeStamp(), priority, createTime, logAggregationContext
);
password = this.CreatePassword(tokenIdentifier);
}
finally
{
this.readLock.Unlock();
}
return(BuilderUtils.NewContainerToken(nodeId, password, tokenIdentifier));
}
/// <exception cref="System.Exception"/>
private LogAggregationContext GetLogAggregationContextFromContainerToken(MockRM rm1
, MockNM nm1, LogAggregationContext logAggregationContext)
{
RMApp app2 = rm1.SubmitApp(200, logAggregationContext);
MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm1);
nm1.NodeHeartbeat(true);
// request a container.
am2.Allocate("127.0.0.1", 512, 1, new AList <ContainerId>());
ContainerId containerId = ContainerId.NewContainerId(am2.GetApplicationAttemptId(
), 2);
rm1.WaitForState(nm1, containerId, RMContainerState.Allocated);
// acquire the container.
IList <Container> containers = am2.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers();
NUnit.Framework.Assert.AreEqual(containerId, containers[0].GetId());
// container token is generated.
NUnit.Framework.Assert.IsNotNull(containers[0].GetContainerToken());
ContainerTokenIdentifier token = BuilderUtils.NewContainerTokenIdentifier(containers
[0].GetContainerToken());
return(token.GetLogAggregationContext());
}