internal WrappedApplication(TestApplication _enclosing, int id, long timestamp, string
user, int numContainers)
{
this._enclosing = _enclosing;
Configuration conf = new Configuration();
this.dispatcher = new DrainDispatcher();
this.containerTokenIdentifierMap = new Dictionary <ContainerId, ContainerTokenIdentifier
>();
this.dispatcher.Init(conf);
this.localizerBus = Org.Mockito.Mockito.Mock <EventHandler>();
this.launcherBus = Org.Mockito.Mockito.Mock <EventHandler>();
this.monitorBus = Org.Mockito.Mockito.Mock <EventHandler>();
this.auxBus = Org.Mockito.Mockito.Mock <EventHandler>();
this.containerBus = Org.Mockito.Mockito.Mock <EventHandler>();
this.logAggregationBus = Org.Mockito.Mockito.Mock <EventHandler>();
this.dispatcher.Register(typeof(LocalizationEventType), this.localizerBus);
this.dispatcher.Register(typeof(ContainersLauncherEventType), this.launcherBus);
this.dispatcher.Register(typeof(ContainersMonitorEventType), this.monitorBus);
this.dispatcher.Register(typeof(AuxServicesEventType), this.auxBus);
this.dispatcher.Register(typeof(ContainerEventType), this.containerBus);
this.dispatcher.Register(typeof(LogHandlerEventType), this.logAggregationBus);
this.nmTokenSecretMgr = Org.Mockito.Mockito.Mock <NMTokenSecretManagerInNM>();
this.context = Org.Mockito.Mockito.Mock <Context>();
Org.Mockito.Mockito.When(this.context.GetContainerTokenSecretManager()).ThenReturn
(new NMContainerTokenSecretManager(conf));
Org.Mockito.Mockito.When(this.context.GetApplicationACLsManager()).ThenReturn(new
ApplicationACLsManager(conf));
Org.Mockito.Mockito.When(this.context.GetNMTokenSecretManager()).ThenReturn(this.
nmTokenSecretMgr);
// Setting master key
MasterKey masterKey = new MasterKeyPBImpl();
masterKey.SetKeyId(123);
masterKey.SetBytes(ByteBuffer.Wrap(new byte[] { (123) }));
this.context.GetContainerTokenSecretManager().SetMasterKey(masterKey);
this.user = user;
this.appId = BuilderUtils.NewApplicationId(timestamp, id);
this.app = new ApplicationImpl(this.dispatcher, this.user, this.appId, null, this
.context);
this.containers = new AList <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
for (int i = 0; i < numContainers; i++)
{
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= this._enclosing.CreateMockedContainer(this.appId, i);
this.containers.AddItem(container);
long currentTime = Runtime.CurrentTimeMillis();
ContainerTokenIdentifier identifier = new ContainerTokenIdentifier(container.GetContainerId
(), string.Empty, string.Empty, null, currentTime + 2000, masterKey.GetKeyId(),
currentTime, Priority.NewInstance(0), 0);
this.containerTokenIdentifierMap[identifier.GetContainerID()] = identifier;
this.context.GetContainerTokenSecretManager().StartContainerSuccessful(identifier
);
NUnit.Framework.Assert.IsFalse(this.context.GetContainerTokenSecretManager().IsValidStartContainerRequest
(identifier));
}
this.dispatcher.Start();
}
/// <exception cref="System.IO.IOException"/>
public override void StoreContainerTokenPreviousMasterKey(MasterKey key)
{
lock (this)
{
MasterKeyPBImpl keypb = (MasterKeyPBImpl)key;
containerTokenState.previousMasterKey = new MasterKeyPBImpl(keypb.GetProto());
}
}
/// <exception cref="System.IO.IOException"/>
public override void StoreNMTokenCurrentMasterKey(MasterKey key)
{
lock (this)
{
MasterKeyPBImpl keypb = (MasterKeyPBImpl)key;
nmTokenState.currentMasterKey = new MasterKeyPBImpl(keypb.GetProto());
}
}
/// <exception cref="System.IO.IOException"/>
public override void StoreNMTokenApplicationMasterKey(ApplicationAttemptId attempt
, MasterKey key)
{
lock (this)
{
MasterKeyPBImpl keypb = (MasterKeyPBImpl)key;
nmTokenState.applicationMasterKeys[attempt] = new MasterKeyPBImpl(keypb.GetProto(
));
}
}
public virtual void TestMasterKeyPBImpl()
{
MasterKeyPBImpl original = new MasterKeyPBImpl();
original.SetBytes(ByteBuffer.Allocate(0));
original.SetKeyId(1);
MasterKeyPBImpl copy = new MasterKeyPBImpl(original.GetProto());
NUnit.Framework.Assert.AreEqual(1, copy.GetKeyId());
NUnit.Framework.Assert.IsTrue(original.Equals(copy));
NUnit.Framework.Assert.AreEqual(original.GetHashCode(), copy.GetHashCode());
}
/// <exception cref="System.IO.IOException"/>
private void StoreMasterKey(string dbKey, MasterKey key)
{
MasterKeyPBImpl pb = (MasterKeyPBImpl)key;
try
{
db.Put(JniDBFactory.Bytes(dbKey), pb.GetProto().ToByteArray());
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual RegisterNodeManagerResponse RegisterNodeManager(RegisterNodeManagerRequest
request)
{
RegisterNodeManagerResponse response = recordFactory.NewRecordInstance <RegisterNodeManagerResponse
>();
MasterKey masterKey = new MasterKeyPBImpl();
masterKey.SetKeyId(123);
masterKey.SetBytes(ByteBuffer.Wrap(new byte[] { 123 }));
response.SetContainerTokenMasterKey(masterKey);
response.SetNMTokenMasterKey(masterKey);
return(response);
}
public virtual void TestApplicationRecovery()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.NmRecoveryEnabled, true);
conf.Set(YarnConfiguration.NmAddress, "localhost:1234");
conf.SetBoolean(YarnConfiguration.YarnAclEnable, true);
conf.Set(YarnConfiguration.YarnAdminAcl, "yarn_admin_user");
NMStateStoreService stateStore = new NMMemoryStateStoreService();
stateStore.Init(conf);
stateStore.Start();
Context context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf
), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore
);
ContainerManagerImpl cm = CreateContainerManager(context);
cm.Init(conf);
cm.Start();
// simulate registration with RM
MasterKey masterKey = new MasterKeyPBImpl();
masterKey.SetKeyId(123);
masterKey.SetBytes(ByteBuffer.Wrap(new byte[] { 123 }));
context.GetContainerTokenSecretManager().SetMasterKey(masterKey);
context.GetNMTokenSecretManager().SetMasterKey(masterKey);
// add an application by starting a container
string appUser = "******";
string modUser = "******";
string viewUser = "******";
string enemyUser = "******";
ApplicationId appId = ApplicationId.NewInstance(0, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
ContainerId cid = ContainerId.NewContainerId(attemptId, 1);
IDictionary <string, LocalResource> localResources = Collections.EmptyMap();
IDictionary <string, string> containerEnv = Sharpen.Collections.EmptyMap();
IList <string> containerCmds = Sharpen.Collections.EmptyList();
IDictionary <string, ByteBuffer> serviceData = Sharpen.Collections.EmptyMap();
Credentials containerCreds = new Credentials();
DataOutputBuffer dob = new DataOutputBuffer();
containerCreds.WriteTokenStorageToStream(dob);
ByteBuffer containerTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>();
acls[ApplicationAccessType.ModifyApp] = modUser;
acls[ApplicationAccessType.ViewApp] = viewUser;
ContainerLaunchContext clc = ContainerLaunchContext.NewInstance(localResources, containerEnv
, containerCmds, serviceData, containerTokens, acls);
// create the logAggregationContext
LogAggregationContext logAggregationContext = LogAggregationContext.NewInstance("includePattern"
, "excludePattern", "includePatternInRollingAggregation", "excludePatternInRollingAggregation"
);
StartContainersResponse startResponse = StartContainer(context, cm, cid, clc, logAggregationContext
);
NUnit.Framework.Assert.IsTrue(startResponse.GetFailedRequests().IsEmpty());
NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = context.GetApplications()[appId];
NUnit.Framework.Assert.IsNotNull(app);
WaitForAppState(app, ApplicationState.Initing);
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId));
// reset container manager and verify app recovered with proper acls
cm.Stop();
context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new
NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore);
cm = CreateContainerManager(context);
cm.Init(conf);
cm.Start();
NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count);
app = context.GetApplications()[appId];
NUnit.Framework.Assert.IsNotNull(app);
// check whether LogAggregationContext is recovered correctly
LogAggregationContext recovered = ((ApplicationImpl)app).GetLogAggregationContext
();
NUnit.Framework.Assert.IsNotNull(recovered);
NUnit.Framework.Assert.AreEqual(logAggregationContext.GetIncludePattern(), recovered
.GetIncludePattern());
NUnit.Framework.Assert.AreEqual(logAggregationContext.GetExcludePattern(), recovered
.GetExcludePattern());
NUnit.Framework.Assert.AreEqual(logAggregationContext.GetRolledLogsIncludePattern
(), recovered.GetRolledLogsIncludePattern());
NUnit.Framework.Assert.AreEqual(logAggregationContext.GetRolledLogsExcludePattern
(), recovered.GetRolledLogsExcludePattern());
WaitForAppState(app, ApplicationState.Initing);
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId));
// simulate application completion
IList <ApplicationId> finishedApps = new AList <ApplicationId>();
finishedApps.AddItem(appId);
cm.Handle(new CMgrCompletedAppsEvent(finishedApps, CMgrCompletedAppsEvent.Reason.
ByResourcemanager));
WaitForAppState(app, ApplicationState.ApplicationResourcesCleaningup);
// restart and verify app is marked for finishing
cm.Stop();
context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new
NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore);
cm = CreateContainerManager(context);
cm.Init(conf);
cm.Start();
NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count);
app = context.GetApplications()[appId];
NUnit.Framework.Assert.IsNotNull(app);
WaitForAppState(app, ApplicationState.ApplicationResourcesCleaningup);
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId));
// simulate log aggregation completion
app.Handle(new ApplicationEvent(app.GetAppId(), ApplicationEventType.ApplicationResourcesCleanedup
));
NUnit.Framework.Assert.AreEqual(app.GetApplicationState(), ApplicationState.Finished
);
app.Handle(new ApplicationEvent(app.GetAppId(), ApplicationEventType.ApplicationLogHandlingFinished
));
// restart and verify app is no longer present after recovery
cm.Stop();
context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new
NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore);
cm = CreateContainerManager(context);
cm.Init(conf);
cm.Start();
NUnit.Framework.Assert.IsTrue(context.GetApplications().IsEmpty());
cm.Stop();
}
