public virtual void TestRetriesOnFailures()
{
ContainerManagementProtocol mockProxy = Org.Mockito.Mockito.Mock <ContainerManagementProtocol
>();
StartContainersResponse mockResponse = Org.Mockito.Mockito.Mock <StartContainersResponse
>();
Org.Mockito.Mockito.When(mockProxy.StartContainers(Matchers.Any <StartContainersRequest
>())).ThenThrow(new NMNotYetReadyException("foo")).ThenReturn(mockResponse);
Configuration conf = new Configuration();
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
conf.SetInt(YarnConfiguration.ClientNmConnectRetryIntervalMs, 1);
DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm = new _MockRMWithCustomAMLauncher_206(dispatcher, mockProxy, conf, null
);
rm.Start();
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 5120);
RMApp app = rm.SubmitApp(2000);
ApplicationAttemptId appAttemptId = app.GetCurrentAppAttempt().GetAppAttemptId();
// kick the scheduling
nm1.NodeHeartbeat(true);
dispatcher.Await();
rm.WaitForState(appAttemptId, RMAppAttemptState.Launched, 500);
}
public _MockRMWithCustomAMLauncher_206(DrainDispatcher dispatcher, ContainerManagementProtocol
mockProxy, Configuration baseArg1, ContainerManagementProtocol baseArg2)
: base(baseArg1, baseArg2)
{
this.dispatcher = dispatcher;
this.mockProxy = mockProxy;
}
/// <exception cref="System.Exception"/>
private void GetContainerStatus(YarnRPC rpc, Token nmToken, ContainerId containerId
, ApplicationAttemptId appAttemptId, NodeId nodeId, bool isExceptionExpected)
{
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(containerId);
GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds
);
ContainerManagementProtocol proxy = null;
try
{
proxy = GetContainerManagementProtocolProxy(rpc, nmToken, nodeId, appAttemptId.ToString
());
GetContainerStatusesResponse statuses = proxy.GetContainerStatuses(request);
if (statuses.GetFailedRequests() != null && statuses.GetFailedRequests().Contains
(containerId))
{
ParseAndThrowException(statuses.GetFailedRequests()[containerId].DeSerialize());
}
}
finally
{
if (proxy != null)
{
rpc.StopProxy(proxy, conf);
}
}
}
/// <exception cref="System.Exception"/>
private void StartContainer(YarnRPC rpc, Token nmToken, Token containerToken, NodeId
nodeId, string user)
{
ContainerLaunchContext context = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext
>();
StartContainerRequest scRequest = StartContainerRequest.NewInstance(context, containerToken
);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
ContainerManagementProtocol proxy = null;
try
{
proxy = GetContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
StartContainersResponse response = proxy.StartContainers(allRequests);
foreach (SerializedException ex in response.GetFailedRequests().Values)
{
ParseAndThrowException(ex.DeSerialize());
}
}
finally
{
if (proxy != null)
{
rpc.StopProxy(proxy, conf);
}
}
}
/// <exception cref="System.Exception"/>
private void StopContainer(YarnRPC rpc, Token nmToken, IList <ContainerId> containerId
, ApplicationAttemptId appAttemptId, NodeId nodeId)
{
StopContainersRequest request = StopContainersRequest.NewInstance(containerId);
ContainerManagementProtocol proxy = null;
try
{
proxy = GetContainerManagementProtocolProxy(rpc, nmToken, nodeId, appAttemptId.ToString
());
StopContainersResponse response = proxy.StopContainers(request);
if (response.GetFailedRequests() != null && response.GetFailedRequests().Contains
(containerId))
{
ParseAndThrowException(response.GetFailedRequests()[containerId].DeSerialize());
}
}
catch (Exception)
{
if (proxy != null)
{
rpc.StopProxy(proxy, conf);
}
}
}
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public static void WaitForContainerState(ContainerManagementProtocol containerManager
, ContainerId containerID, ContainerState finalState, int timeOutMax)
{
IList <ContainerId> list = new AList <ContainerId>();
list.AddItem(containerID);
GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(list
);
ContainerStatus containerStatus = containerManager.GetContainerStatuses(request).
GetContainerStatuses()[0];
int timeoutSecs = 0;
while (!containerStatus.GetState().Equals(finalState) && timeoutSecs++ < timeOutMax
)
{
Sharpen.Thread.Sleep(1000);
Log.Info("Waiting for container to get into state " + finalState + ". Current state is "
+ containerStatus.GetState());
containerStatus = containerManager.GetContainerStatuses(request).GetContainerStatuses
()[0];
}
Log.Info("Container state is " + containerStatus.GetState());
NUnit.Framework.Assert.AreEqual("ContainerState is not correct (timedout)", finalState
, containerStatus.GetState());
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public static void StartContainer(NodeManager nm, ContainerId cId, FileContext localFS
, FilePath scriptFileDir, FilePath processStartFile)
{
FilePath scriptFile = CreateUnhaltingScriptFile(cId, scriptFileDir, processStartFile
);
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
NodeId nodeId = BuilderUtils.NewNodeId(Sharpen.Extensions.GetAddressByName("localhost"
).ToString(), 12345);
URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(scriptFile.GetAbsolutePath())));
LocalResource localResource = recordFactory.NewRecordInstance <LocalResource>();
localResource.SetResource(localResourceUri);
localResource.SetSize(-1);
localResource.SetVisibility(LocalResourceVisibility.Application);
localResource.SetType(LocalResourceType.File);
localResource.SetTimestamp(scriptFile.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = localResource;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile));
containerLaunchContext.SetCommands(commands);
IPEndPoint containerManagerBindAddress = NetUtils.CreateSocketAddrForHost("127.0.0.1"
, 12345);
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.ToString
());
Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> nmToken = ConverterUtils
.ConvertFromYarn(nm.GetNMContext().GetNMTokenSecretManager().CreateNMToken(cId.GetApplicationAttemptId
(), nodeId, user), containerManagerBindAddress);
currentUser.AddToken(nmToken);
ContainerManagementProtocol containerManager = currentUser.DoAs(new _PrivilegedAction_229
());
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, TestContainerManager.CreateContainerToken(cId, 0, nodeId, user, nm.GetNMContext
().GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds
);
ContainerStatus containerStatus = containerManager.GetContainerStatuses(request).
GetContainerStatuses()[0];
NUnit.Framework.Assert.AreEqual(ContainerState.Running, containerStatus.GetState(
));
}
/// <summary>This tests whether a containerId is serialized/deserialized with epoch.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private void TestContainerTokenWithEpoch(Configuration conf)
{
Log.Info("Running test for serializing/deserializing containerIds");
NMTokenSecretManagerInRM nmTokenSecretManagerInRM = yarnCluster.GetResourceManager
().GetRMContext().GetNMTokenSecretManager();
ApplicationId appId = ApplicationId.NewInstance(1, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 0);
ContainerId cId = ContainerId.NewContainerId(appAttemptId, (5L << 40) | 3L);
NodeManager nm = yarnCluster.GetNodeManager(0);
NMTokenSecretManagerInNM nmTokenSecretManagerInNM = nm.GetNMContext().GetNMTokenSecretManager
();
string user = "******";
WaitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm);
NodeId nodeId = nm.GetNMContext().GetNodeId();
// Both id should be equal.
NUnit.Framework.Assert.AreEqual(nmTokenSecretManagerInNM.GetCurrentKey().GetKeyId
(), nmTokenSecretManagerInRM.GetCurrentKey().GetKeyId());
// Creating a normal Container Token
RMContainerTokenSecretManager containerTokenSecretManager = yarnCluster.GetResourceManager
().GetRMContext().GetContainerTokenSecretManager();
Resource r = Resource.NewInstance(1230, 2);
Org.Apache.Hadoop.Yarn.Api.Records.Token containerToken = containerTokenSecretManager
.CreateContainerToken(cId, nodeId, user, r, Priority.NewInstance(0), 0);
ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
);
byte[] tokenIdentifierContent = ((byte[])containerToken.GetIdentifier().Array());
DataInputBuffer dib = new DataInputBuffer();
dib.Reset(tokenIdentifierContent, tokenIdentifierContent.Length);
containerTokenIdentifier.ReadFields(dib);
NUnit.Framework.Assert.AreEqual(cId, containerTokenIdentifier.GetContainerID());
NUnit.Framework.Assert.AreEqual(cId.ToString(), containerTokenIdentifier.GetContainerID
().ToString());
Org.Apache.Hadoop.Yarn.Api.Records.Token nmToken = nmTokenSecretManagerInRM.CreateNMToken
(appAttemptId, nodeId, user);
YarnRPC rpc = YarnRPC.Create(conf);
TestStartContainer(rpc, appAttemptId, nodeId, containerToken, nmToken, false);
IList <ContainerId> containerIds = new List <ContainerId>();
containerIds.AddItem(cId);
ContainerManagementProtocol proxy = GetContainerManagementProtocolProxy(rpc, nmToken
, nodeId, user);
GetContainerStatusesResponse res = proxy.GetContainerStatuses(GetContainerStatusesRequest
.NewInstance(containerIds));
NUnit.Framework.Assert.IsNotNull(res.GetContainerStatuses()[0]);
NUnit.Framework.Assert.AreEqual(cId, res.GetContainerStatuses()[0].GetContainerId
());
NUnit.Framework.Assert.AreEqual(cId.ToString(), res.GetContainerStatuses()[0].GetContainerId
().ToString());
}
/// <exception cref="System.Exception"/>
private void TestRPCTimeout(string rpcClass)
{
Configuration conf = new Configuration();
// set timeout low for the test
conf.SetInt("yarn.rpc.nm-command-timeout", 3000);
conf.Set(YarnConfiguration.IpcRpcImpl, rpcClass);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestContainerLaunchRPC.DummyContainerManager
(this), addr, conf, null, 1);
server.Start();
try
{
ContainerManagementProtocol proxy = (ContainerManagementProtocol)rpc.GetProxy(typeof(
ContainerManagementProtocol), server.GetListenerAddress(), conf);
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
ApplicationId applicationId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 0);
ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 100);
NodeId nodeId = NodeId.NewInstance("localhost", 1234);
Resource resource = Resource.NewInstance(1234, 2);
ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
containerId, "localhost", "user", resource, Runtime.CurrentTimeMillis() + 10000,
42, 42, Priority.NewInstance(0), 0);
Token containerToken = TestRPC.NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString
("password"), containerTokenIdentifier);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
try
{
proxy.StartContainers(allRequests);
}
catch (Exception e)
{
Log.Info(StringUtils.StringifyException(e));
NUnit.Framework.Assert.AreEqual("Error, exception is not: " + typeof(SocketTimeoutException
).FullName, typeof(SocketTimeoutException).FullName, e.GetType().FullName);
return;
}
}
finally
{
server.Stop();
}
NUnit.Framework.Assert.Fail("timeout exception should have occurred!");
}
public ContainerManagementProtocolProxyData(ContainerManagementProtocolProxy _enclosing
, YarnRPC rpc, string containerManagerBindAddr, ContainerId containerId, Token token
)
{
this._enclosing = _enclosing;
this.containerManagerBindAddr = containerManagerBindAddr;
this.activeCallers = 0;
this.scheduledForClose = false;
this.token = token;
this.proxy = this.NewProxy(rpc, containerManagerBindAddr, containerId, token);
}
/// <exception cref="System.Exception"/>
public virtual void TestNMProxyRetry()
{
containerManager.Start();
containerManager.SetBlockNewContainerRequests(false);
StartContainersRequest allRequests = Records.NewRecord <StartContainersRequest>();
ApplicationId appId = ApplicationId.NewInstance(1, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
Token nmToken = context.GetNMTokenSecretManager().CreateNMToken(attemptId, context
.GetNodeId(), user);
IPEndPoint address = conf.GetSocketAddr(YarnConfiguration.NmBindHost, YarnConfiguration
.NmAddress, YarnConfiguration.DefaultNmAddress, YarnConfiguration.DefaultNmPort);
Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> token = ConverterUtils.
ConvertFromYarn(nmToken, SecurityUtil.BuildTokenService(address));
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(user);
ugi.AddToken(token);
ContainerManagementProtocol proxy = NMProxy.CreateNMProxy <ContainerManagementProtocol
>(conf, ugi, YarnRPC.Create(conf), address);
retryCount = 0;
shouldThrowNMNotYetReadyException = false;
proxy.StartContainers(allRequests);
NUnit.Framework.Assert.AreEqual(5, retryCount);
retryCount = 0;
shouldThrowNMNotYetReadyException = false;
proxy.StopContainers(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <StopContainersRequest
>());
NUnit.Framework.Assert.AreEqual(5, retryCount);
retryCount = 0;
shouldThrowNMNotYetReadyException = false;
proxy.GetContainerStatuses(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetContainerStatusesRequest
>());
NUnit.Framework.Assert.AreEqual(5, retryCount);
retryCount = 0;
shouldThrowNMNotYetReadyException = true;
proxy.StartContainers(allRequests);
NUnit.Framework.Assert.AreEqual(5, retryCount);
}
public MockRMWithCustomAMLauncher(ContainerManagementProtocol containerManager)
: this(new Configuration(), containerManager)
{
}
public ContainerLauncherImplUnderTest(AppContext context, ContainerManagementProtocol
containerManager)
: base(context)
{
this.containerManager = containerManager;
}
/// <exception cref="System.Exception"/>
private void Test(string rpcClass)
{
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.IpcRpcImpl, rpcClass);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestRPC.DummyContainerManager
(this), addr, conf, null, 1);
server.Start();
RPC.SetProtocolEngine(conf, typeof(ContainerManagementProtocolPB), typeof(ProtobufRpcEngine
));
ContainerManagementProtocol proxy = (ContainerManagementProtocol)rpc.GetProxy(typeof(
ContainerManagementProtocol), NetUtils.GetConnectAddress(server), conf);
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
ApplicationId applicationId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 0);
ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 100);
NodeId nodeId = NodeId.NewInstance("localhost", 1234);
Resource resource = Resource.NewInstance(1234, 2);
ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
containerId, "localhost", "user", resource, Runtime.CurrentTimeMillis() + 10000,
42, 42, Priority.NewInstance(0), 0);
Token containerToken = NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString
("password"), containerTokenIdentifier);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
proxy.StartContainers(allRequests);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(containerId);
GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance(
containerIds);
GetContainerStatusesResponse response = proxy.GetContainerStatuses(gcsRequest);
IList <ContainerStatus> statuses = response.GetContainerStatuses();
//test remote exception
bool exception = false;
try
{
StopContainersRequest stopRequest = recordFactory.NewRecordInstance <StopContainersRequest
>();
stopRequest.SetContainerIds(containerIds);
proxy.StopContainers(stopRequest);
}
catch (YarnException e)
{
exception = true;
NUnit.Framework.Assert.IsTrue(e.Message.Contains(ExceptionMsg));
NUnit.Framework.Assert.IsTrue(e.Message.Contains(ExceptionCause));
System.Console.Out.WriteLine("Test Exception is " + e.Message);
}
catch (Exception ex)
{
Sharpen.Runtime.PrintStackTrace(ex);
}
NUnit.Framework.Assert.IsTrue(exception);
server.Stop();
NUnit.Framework.Assert.IsNotNull(statuses[0]);
NUnit.Framework.Assert.AreEqual(ContainerState.Running, statuses[0].GetState());
}
/// <exception cref="System.Exception"/>
public virtual void TestClientTokenRace()
{
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
ContainerManagementProtocol containerManager = Org.Mockito.Mockito.Mock <ContainerManagementProtocol
>();
StartContainersResponse mockResponse = Org.Mockito.Mockito.Mock <StartContainersResponse
>();
Org.Mockito.Mockito.When(containerManager.StartContainers((StartContainersRequest
)Matchers.Any())).ThenReturn(mockResponse);
DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm = new _MockRMWithCustomAMLauncher_433(dispatcher, conf, containerManager
);
rm.Start();
// Submit an app
RMApp app = rm.SubmitApp(1024);
// Set up a node.
MockNM nm1 = rm.RegisterNode("localhost:1234", 3072);
nm1.NodeHeartbeat(true);
dispatcher.Await();
nm1.NodeHeartbeat(true);
dispatcher.Await();
ApplicationAttemptId appAttempt = app.GetCurrentAppAttempt().GetAppAttemptId();
MockAM mockAM = new MockAM(rm.GetRMContext(), rm.GetApplicationMasterService(), app
.GetCurrentAppAttempt().GetAppAttemptId());
UserGroupInformation appUgi = UserGroupInformation.CreateRemoteUser(appAttempt.ToString
());
RegisterApplicationMasterResponse response = appUgi.DoAs(new _PrivilegedAction_469
(mockAM));
// Get the app-report.
GetApplicationReportRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<GetApplicationReportRequest>();
request.SetApplicationId(app.GetApplicationId());
GetApplicationReportResponse reportResponse = rm.GetClientRMService().GetApplicationReport
(request);
ApplicationReport appReport = reportResponse.GetApplicationReport();
Org.Apache.Hadoop.Yarn.Api.Records.Token originalClientToAMToken = appReport.GetClientToAMToken
();
// ClientToAMToken master key should have been received on register
// application master response.
ByteBuffer clientMasterKey = response.GetClientToAMTokenMasterKey();
NUnit.Framework.Assert.IsNotNull(clientMasterKey);
NUnit.Framework.Assert.IsTrue(((byte[])clientMasterKey.Array()).Length > 0);
// Start the AM with the correct shared-secret.
ApplicationAttemptId appAttemptId = app.GetAppAttempts().Keys.GetEnumerator().Next
();
NUnit.Framework.Assert.IsNotNull(appAttemptId);
TestClientToAMTokens.CustomAM am = new TestClientToAMTokens.CustomAM(appAttemptId
, null);
am.Init(conf);
am.Start();
// Now the real test!
// Set up clients to be able to pick up correct tokens.
SecurityUtil.SetSecurityInfoProviders(new TestClientToAMTokens.CustomSecurityInfo
());
Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> token = ConverterUtils
.ConvertFromYarn(originalClientToAMToken, am.address);
// Schedule the key to be set after a significant delay
Timer timer = new Timer();
TimerTask timerTask = new _TimerTask_516(am, clientMasterKey);
timer.Schedule(timerTask, 250);
// connect should pause waiting for the master key to arrive
VerifyValidToken(conf, am, token);
am.Stop();
rm.Stop();
}
public virtual void TestClientToAMTokens()
{
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
ContainerManagementProtocol containerManager = Org.Mockito.Mockito.Mock <ContainerManagementProtocol
>();
StartContainersResponse mockResponse = Org.Mockito.Mockito.Mock <StartContainersResponse
>();
Org.Mockito.Mockito.When(containerManager.StartContainers((StartContainersRequest
)Matchers.Any())).ThenReturn(mockResponse);
DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm = new _MockRMWithCustomAMLauncher_192(dispatcher, conf, containerManager
);
rm.Start();
// Submit an app
RMApp app = rm.SubmitApp(1024);
// Set up a node.
MockNM nm1 = rm.RegisterNode("localhost:1234", 3072);
nm1.NodeHeartbeat(true);
dispatcher.Await();
nm1.NodeHeartbeat(true);
dispatcher.Await();
ApplicationAttemptId appAttempt = app.GetCurrentAppAttempt().GetAppAttemptId();
MockAM mockAM = new MockAM(rm.GetRMContext(), rm.GetApplicationMasterService(), app
.GetCurrentAppAttempt().GetAppAttemptId());
UserGroupInformation appUgi = UserGroupInformation.CreateRemoteUser(appAttempt.ToString
());
RegisterApplicationMasterResponse response = appUgi.DoAs(new _PrivilegedAction_229
(mockAM));
// Get the app-report.
GetApplicationReportRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<GetApplicationReportRequest>();
request.SetApplicationId(app.GetApplicationId());
GetApplicationReportResponse reportResponse = rm.GetClientRMService().GetApplicationReport
(request);
ApplicationReport appReport = reportResponse.GetApplicationReport();
Org.Apache.Hadoop.Yarn.Api.Records.Token originalClientToAMToken = appReport.GetClientToAMToken
();
// ClientToAMToken master key should have been received on register
// application master response.
NUnit.Framework.Assert.IsNotNull(response.GetClientToAMTokenMasterKey());
NUnit.Framework.Assert.IsTrue(((byte[])response.GetClientToAMTokenMasterKey().Array
()).Length > 0);
// Start the AM with the correct shared-secret.
ApplicationAttemptId appAttemptId = app.GetAppAttempts().Keys.GetEnumerator().Next
();
NUnit.Framework.Assert.IsNotNull(appAttemptId);
TestClientToAMTokens.CustomAM am = new TestClientToAMTokens.CustomAM(appAttemptId
, ((byte[])response.GetClientToAMTokenMasterKey().Array()));
am.Init(conf);
am.Start();
// Now the real test!
// Set up clients to be able to pick up correct tokens.
SecurityUtil.SetSecurityInfoProviders(new TestClientToAMTokens.CustomSecurityInfo
());
// Verify denial for unauthenticated user
try
{
TestClientToAMTokens.CustomProtocol client = (TestClientToAMTokens.CustomProtocol
)RPC.GetProxy <TestClientToAMTokens.CustomProtocol>(1L, am.address, conf);
client.Ping();
NUnit.Framework.Assert.Fail("Access by unauthenticated user should fail!!");
}
catch (Exception)
{
NUnit.Framework.Assert.IsFalse(am.pinged);
}
Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> token = ConverterUtils
.ConvertFromYarn(originalClientToAMToken, am.address);
// Verify denial for a malicious user with tampered ID
VerifyTokenWithTamperedID(conf, am, token);
// Verify denial for a malicious user with tampered user-name
VerifyTokenWithTamperedUserName(conf, am, token);
// Now for an authenticated user
VerifyValidToken(conf, am, token);
// Verify for a new version token
VerifyNewVersionToken(conf, am, token, rm);
am.Stop();
rm.Stop();
}
public _AMLauncher_213(ContainerManagementProtocol mockProxy, RMContext baseArg1,
RMAppAttempt baseArg2, AMLauncherEventType baseArg3, Configuration baseArg4)
: base(baseArg1, baseArg2, baseArg3, baseArg4)
{
this.mockProxy = mockProxy;
}
public MockRMWithAMS(Configuration conf, ContainerManagementProtocol containerManager
)
: base(conf, containerManager)
{
}
public MockRMWithCustomAMLauncher(Configuration conf, ContainerManagementProtocol
containerManager)
: base(conf)
{
this.containerManager = containerManager;
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.Exception"/>
public virtual void TestClearLocalDirWhenNodeReboot()
{
nm = new TestNodeManagerReboot.MyNodeManager(this);
nm.Start();
ContainerManagementProtocol containerManager = nm.GetContainerManager();
// create files under fileCache
CreateFiles(nmLocalDir.GetAbsolutePath(), ContainerLocalizer.Filecache, 100);
localResourceDir.Mkdirs();
ContainerLaunchContext containerLaunchContext = Records.NewRecord <ContainerLaunchContext
>();
// Construct the Container-id
ContainerId cId = CreateContainerId();
URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(localResourceDir.GetAbsolutePath())));
LocalResource localResource = LocalResource.NewInstance(localResourceUri, LocalResourceType
.File, LocalResourceVisibility.Application, -1, localResourceDir.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = localResource;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = new AList <string>();
containerLaunchContext.SetCommands(commands);
NodeId nodeId = nm.GetNMContext().GetNodeId();
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, TestContainerManager.CreateContainerToken(cId, 0, nodeId, destinationFile, nm.
GetNMContext().GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.GetApplicationAttemptId
().ToString());
NMTokenIdentifier nmIdentifier = new NMTokenIdentifier(cId.GetApplicationAttemptId
(), nodeId, user, 123);
currentUser.AddTokenIdentifier(nmIdentifier);
currentUser.DoAs(new _PrivilegedExceptionAction_152(this, allRequests));
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds
);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= nm.GetNMContext().GetContainers()[request.GetContainerIds()[0]];
int MaxTries = 20;
int numTries = 0;
while (!container.GetContainerState().Equals(ContainerState.Done) && numTries <=
MaxTries)
{
try
{
Sharpen.Thread.Sleep(500);
}
catch (Exception)
{
}
// Do nothing
numTries++;
}
NUnit.Framework.Assert.AreEqual(ContainerState.Done, container.GetContainerState(
));
NUnit.Framework.Assert.IsTrue("The container should create a subDir named currentUser: "******"under localDir/usercache", NumOfLocalDirs(nmLocalDir.GetAbsolutePath(
), ContainerLocalizer.Usercache) > 0);
NUnit.Framework.Assert.IsTrue("There should be files or Dirs under nm_private when "
+ "container is launched", NumOfLocalDirs(nmLocalDir.GetAbsolutePath(), ResourceLocalizationService
.NmPrivateDir) > 0);
// restart the NodeManager
RestartNM(MaxTries);
CheckNumOfLocalDirs();
Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).Delete((string
)Matchers.IsNull(), Matchers.ArgThat(new TestNodeManagerReboot.PathInclude(this,
ResourceLocalizationService.NmPrivateDir + "_DEL_")));
Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).Delete((string
)Matchers.IsNull(), Matchers.ArgThat(new TestNodeManagerReboot.PathInclude(this,
ContainerLocalizer.Filecache + "_DEL_")));
Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).ScheduleFileDeletionTask
(Matchers.ArgThat(new TestNodeManagerReboot.FileDeletionInclude(this, user, null
, new string[] { destinationFile })));
Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).ScheduleFileDeletionTask
(Matchers.ArgThat(new TestNodeManagerReboot.FileDeletionInclude(this, null, ContainerLocalizer
.Usercache + "_DEL_", new string[] { })));
// restart the NodeManager again
// this time usercache directory should be empty
RestartNM(MaxTries);
CheckNumOfLocalDirs();
}
/// <exception cref="System.IO.IOException"/>
private void Connect()
{
ContainerId masterContainerID = masterContainer.GetId();
containerMgrProxy = GetContainerMgrProxy(masterContainerID);
}
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public static void WaitForContainerState(ContainerManagementProtocol containerManager
, ContainerId containerID, ContainerState finalState)
{
WaitForContainerState(containerManager, containerID, finalState, 20);
}
public _ApplicationMasterLauncher_209(ContainerManagementProtocol mockProxy, RMContext
baseArg1)
: base(baseArg1)
{
this.mockProxy = mockProxy;
}
public virtual void SetContainerManager(ContainerManagementProtocol containerManager
)
{
this.containerManager = containerManager;
}
public ContainerManagementProtocolPBServiceImpl(ContainerManagementProtocol impl)
{
this.real = impl;
}
