/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual StartContainersResponse StartContainers(StartContainersRequest requests
)
{
StartContainersResponse response = TestRPC.recordFactory.NewRecordInstance <StartContainersResponse
>();
foreach (StartContainerRequest request in requests.GetStartContainerRequests())
{
Token containerToken = request.GetContainerToken();
ContainerTokenIdentifier tokenId = null;
try
{
tokenId = TestRPC.NewContainerTokenIdentifier(containerToken);
}
catch (IOException e)
{
throw RPCUtil.GetRemoteException(e);
}
ContainerStatus status = TestRPC.recordFactory.NewRecordInstance <ContainerStatus>
();
status.SetState(ContainerState.Running);
status.SetContainerId(tokenId.GetContainerID());
status.SetExitStatus(0);
this.statuses.AddItem(status);
}
return(response);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public StartContainersResponse StartContainers(StartContainersRequest requests)
{
StartContainerRequest request = requests.GetStartContainerRequests()[0];
Log.Info("Container started by MyContainerManager: " + request);
launched = true;
IDictionary <string, string> env = request.GetContainerLaunchContext().GetEnvironment
();
Token containerToken = request.GetContainerToken();
ContainerTokenIdentifier tokenId = null;
try
{
tokenId = BuilderUtils.NewContainerTokenIdentifier(containerToken);
}
catch (IOException e)
{
throw RPCUtil.GetRemoteException(e);
}
ContainerId containerId = tokenId.GetContainerID();
containerIdAtContainerManager = containerId.ToString();
attemptIdAtContainerManager = containerId.GetApplicationAttemptId().ToString();
nmHostAtContainerManager = tokenId.GetNmHostAddress();
submitTimeAtContainerManager = long.Parse(env[ApplicationConstants.AppSubmitTimeEnv
]);
maxAppAttempts = System.Convert.ToInt32(env[ApplicationConstants.MaxAppAttemptsEnv
]);
return(StartContainersResponse.NewInstance(new Dictionary <string, ByteBuffer>(),
new AList <ContainerId>(), new Dictionary <ContainerId, SerializedException>()));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private void Launch()
{
Connect();
ContainerId masterContainerID = masterContainer.GetId();
ApplicationSubmissionContext applicationContext = application.GetSubmissionContext
();
Log.Info("Setting up container " + masterContainer + " for AM " + application.GetAppAttemptId
());
ContainerLaunchContext launchContext = CreateAMContainerLaunchContext(applicationContext
, masterContainerID);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(launchContext
, masterContainer.GetContainerToken());
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
StartContainersResponse response = containerMgrProxy.StartContainers(allRequests);
if (response.GetFailedRequests() != null && response.GetFailedRequests().Contains
(masterContainerID))
{
Exception t = response.GetFailedRequests()[masterContainerID].DeSerialize();
ParseAndThrowException(t);
}
else
{
Log.Info("Done launching container " + masterContainer + " for AM " + application
.GetAppAttemptId());
}
}
/// <exception cref="System.IO.IOException"/>
public virtual StartContainersResponse StartContainers(StartContainersRequest requests
)
{
StartContainerRequest request = requests.GetStartContainerRequests()[0];
ContainerTokenIdentifier containerTokenIdentifier = MRApp.NewContainerTokenIdentifier
(request.GetContainerToken());
// Validate that the container is what RM is giving.
NUnit.Framework.Assert.AreEqual(MRApp.NmHost + ":" + MRApp.NmPort, containerTokenIdentifier
.GetNmHostAddress());
StartContainersResponse response = TestContainerLauncher.recordFactory.NewRecordInstance
<StartContainersResponse>();
this.status = TestContainerLauncher.recordFactory.NewRecordInstance<ContainerStatus
>();
try
{
// make the thread sleep to look like its not going to respond
Sharpen.Thread.Sleep(15000);
}
catch (Exception e)
{
TestContainerLauncher.Log.Error(e);
throw new UndeclaredThrowableException(e);
}
this.status.SetState(ContainerState.Running);
this.status.SetContainerId(containerTokenIdentifier.GetContainerID());
this.status.SetExitStatus(0);
return response;
}
/// <exception cref="System.Exception"/>
private void StartContainer(YarnRPC rpc, Token nmToken, Token containerToken, NodeId
nodeId, string user)
{
ContainerLaunchContext context = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext
>();
StartContainerRequest scRequest = StartContainerRequest.NewInstance(context, containerToken
);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
ContainerManagementProtocol proxy = null;
try
{
proxy = GetContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
StartContainersResponse response = proxy.StartContainers(allRequests);
foreach (SerializedException ex in response.GetFailedRequests().Values)
{
ParseAndThrowException(ex.DeSerialize());
}
}
finally
{
if (proxy != null)
{
rpc.StopProxy(proxy, conf);
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public static void StartContainer(NodeManager nm, ContainerId cId, FileContext localFS
, FilePath scriptFileDir, FilePath processStartFile)
{
FilePath scriptFile = CreateUnhaltingScriptFile(cId, scriptFileDir, processStartFile
);
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
NodeId nodeId = BuilderUtils.NewNodeId(Sharpen.Extensions.GetAddressByName("localhost"
).ToString(), 12345);
URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(scriptFile.GetAbsolutePath())));
LocalResource localResource = recordFactory.NewRecordInstance <LocalResource>();
localResource.SetResource(localResourceUri);
localResource.SetSize(-1);
localResource.SetVisibility(LocalResourceVisibility.Application);
localResource.SetType(LocalResourceType.File);
localResource.SetTimestamp(scriptFile.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = localResource;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile));
containerLaunchContext.SetCommands(commands);
IPEndPoint containerManagerBindAddress = NetUtils.CreateSocketAddrForHost("127.0.0.1"
, 12345);
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.ToString
());
Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> nmToken = ConverterUtils
.ConvertFromYarn(nm.GetNMContext().GetNMTokenSecretManager().CreateNMToken(cId.GetApplicationAttemptId
(), nodeId, user), containerManagerBindAddress);
currentUser.AddToken(nmToken);
ContainerManagementProtocol containerManager = currentUser.DoAs(new _PrivilegedAction_229
());
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, TestContainerManager.CreateContainerToken(cId, 0, nodeId, user, nm.GetNMContext
().GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds
);
ContainerStatus containerStatus = containerManager.GetContainerStatuses(request).
GetContainerStatuses()[0];
NUnit.Framework.Assert.AreEqual(ContainerState.Running, containerStatus.GetState(
));
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual StartContainersResponse StartContainers(StartContainersRequest requests
)
{
lock (this)
{
foreach (StartContainerRequest request in requests.GetStartContainerRequests())
{
Token containerToken = request.GetContainerToken();
ContainerTokenIdentifier tokenId = null;
try
{
tokenId = BuilderUtils.NewContainerTokenIdentifier(containerToken);
}
catch (IOException e)
{
throw RPCUtil.GetRemoteException(e);
}
ContainerId containerID = tokenId.GetContainerID();
ApplicationId applicationId = containerID.GetApplicationAttemptId().GetApplicationId
();
IList <Container> applicationContainers = containers[applicationId];
if (applicationContainers == null)
{
applicationContainers = new AList <Container>();
containers[applicationId] = applicationContainers;
}
// Sanity check
foreach (Container container in applicationContainers)
{
if (container.GetId().CompareTo(containerID) == 0)
{
throw new InvalidOperationException("Container " + containerID + " already setup on node "
+ containerManagerAddress);
}
}
Container container_1 = BuilderUtils.NewContainer(containerID, this.nodeId, nodeHttpAddress
, tokenId.GetResource(), null, null);
// DKDC - Doesn't matter
ContainerStatus containerStatus = BuilderUtils.NewContainerStatus(container_1.GetId
(), ContainerState.New, string.Empty, -1000);
applicationContainers.AddItem(container_1);
containerStatusMap[container_1] = containerStatus;
Resources.SubtractFrom(available, tokenId.GetResource());
Resources.AddTo(used, tokenId.GetResource());
if (Log.IsDebugEnabled())
{
Log.Debug("startContainer:" + " node=" + containerManagerAddress + " application="
+ applicationId + " container=" + container_1 + " available=" + available + " used="
+ used);
}
}
StartContainersResponse response = StartContainersResponse.NewInstance(null, null
, null);
return(response);
}
}
/// <exception cref="System.Exception"/>
private void TestRPCTimeout(string rpcClass)
{
Configuration conf = new Configuration();
// set timeout low for the test
conf.SetInt("yarn.rpc.nm-command-timeout", 3000);
conf.Set(YarnConfiguration.IpcRpcImpl, rpcClass);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestContainerLaunchRPC.DummyContainerManager
(this), addr, conf, null, 1);
server.Start();
try
{
ContainerManagementProtocol proxy = (ContainerManagementProtocol)rpc.GetProxy(typeof(
ContainerManagementProtocol), server.GetListenerAddress(), conf);
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
ApplicationId applicationId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 0);
ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 100);
NodeId nodeId = NodeId.NewInstance("localhost", 1234);
Resource resource = Resource.NewInstance(1234, 2);
ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
containerId, "localhost", "user", resource, Runtime.CurrentTimeMillis() + 10000,
42, 42, Priority.NewInstance(0), 0);
Token containerToken = TestRPC.NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString
("password"), containerTokenIdentifier);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
try
{
proxy.StartContainers(allRequests);
}
catch (Exception e)
{
Log.Info(StringUtils.StringifyException(e));
NUnit.Framework.Assert.AreEqual("Error, exception is not: " + typeof(SocketTimeoutException
).FullName, typeof(SocketTimeoutException).FullName, e.GetType().FullName);
return;
}
}
finally
{
server.Stop();
}
NUnit.Framework.Assert.Fail("timeout exception should have occurred!");
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual StartContainersResponse StartContainers(StartContainersRequest requests
)
{
try
{
// make the thread sleep to look like its not going to respond
Sharpen.Thread.Sleep(10000);
}
catch (Exception e)
{
TestContainerLaunchRPC.Log.Error(e);
throw new YarnException(e);
}
throw new YarnException("Shouldn't happen!!");
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual StartContainersResponse StartContainers(StartContainersRequest requests
)
{
YarnServiceProtos.StartContainersRequestProto requestProto = ((StartContainersRequestPBImpl
)requests).GetProto();
try
{
return(new StartContainersResponsePBImpl(proxy.StartContainers(null, requestProto
)));
}
catch (ServiceException e)
{
RPCUtil.UnwrapAndThrowException(e);
return(null);
}
}
public virtual void TestMultipleContainersLaunch()
{
containerManager.Start();
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
for (int i = 0; i < 10; i++)
{
ContainerId cId = CreateContainerId(i);
long identifier = 0;
if ((i & 1) == 0)
{
// container with even id fail
identifier = ResourceManagerConstants.RmInvalidIdentifier;
}
else
{
identifier = DummyRmIdentifier;
}
Token containerToken = CreateContainerToken(cId, identifier, context.GetNodeId(),
user, context.GetContainerTokenSecretManager());
StartContainerRequest request = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
list.AddItem(request);
}
StartContainersRequest requestList = StartContainersRequest.NewInstance(list);
StartContainersResponse response = containerManager.StartContainers(requestList);
NUnit.Framework.Assert.AreEqual(5, response.GetSuccessfullyStartedContainers().Count
);
foreach (ContainerId id in response.GetSuccessfullyStartedContainers())
{
// Containers with odd id should succeed.
NUnit.Framework.Assert.AreEqual(1, id.GetContainerId() & 1);
}
NUnit.Framework.Assert.AreEqual(5, response.GetFailedRequests().Count);
foreach (KeyValuePair <ContainerId, SerializedException> entry in response.GetFailedRequests
())
{
// Containers with even id should fail.
NUnit.Framework.Assert.AreEqual(0, entry.Key.GetContainerId() & 1);
NUnit.Framework.Assert.IsTrue(entry.Value.GetMessage().Contains("Container " + entry
.Key + " rejected as it is allocated by a previous RM"));
}
}
public override void Run()
{
int numContainers = 0;
int numContainersRejected = 0;
ContainerLaunchContext containerLaunchContext = TestNodeManagerResync.recordFactory
.NewRecordInstance <ContainerLaunchContext>();
try
{
while (!this.isStopped && numContainers < 10)
{
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, null);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
System.Console.Out.WriteLine("no. of containers to be launched: " + numContainers
);
numContainers++;
try
{
this._enclosing.GetContainerManager().StartContainers(allRequests);
}
catch (YarnException e)
{
numContainersRejected++;
NUnit.Framework.Assert.IsTrue(e.Message.Contains("Rejecting new containers as NodeManager has not"
+ " yet connected with ResourceManager"));
NUnit.Framework.Assert.AreEqual(typeof(NMNotYetReadyException).FullName, e.GetType
().FullName);
}
catch (IOException e)
{
Sharpen.Runtime.PrintStackTrace(e);
this._enclosing._enclosing.assertionFailedInThread.Set(true);
}
}
// no. of containers to be launched should equal to no. of
// containers rejected
NUnit.Framework.Assert.AreEqual(numContainers, numContainersRejected);
}
catch (Exception)
{
this._enclosing._enclosing.assertionFailedInThread.Set(true);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private void Assign(Priority priority, NodeType type, IList <Container> containers
)
{
lock (this)
{
for (IEnumerator <Container> i = containers.GetEnumerator(); i.HasNext();)
{
Container container = i.Next();
string host = container.GetNodeId().ToString();
if (Resources.Equals(requestSpec[priority], container.GetResource()))
{
// See which task can use this container
for (IEnumerator <Task> t = tasks[priority].GetEnumerator(); t.HasNext();)
{
Task task = t.Next();
if (task.GetState() == Task.State.Pending && task.CanSchedule(type, host))
{
NodeManager nodeManager = GetNodeManager(host);
task.Start(nodeManager, container.GetId());
i.Remove();
// Track application resource usage
Resources.AddTo(used, container.GetResource());
Log.Info("Assigned container (" + container + ") of type " + type + " to task " +
task.GetTaskId() + " at priority " + priority + " on node " + nodeManager.GetHostName
() + ", currently using " + used + " resources");
// Update resource requests
UpdateResourceRequests(requests[priority], type, task);
// Launch the container
StartContainerRequest scRequest = StartContainerRequest.NewInstance(CreateCLC(),
container.GetContainerToken());
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
nodeManager.StartContainers(allRequests);
break;
}
}
}
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual StartContainersResponse StartContainers(StartContainersRequest request
)
{
try
{
startLaunchBarrier.Await();
completeLaunchBarrier.Await();
//To ensure the kill is started before the launch
Sharpen.Thread.Sleep(100);
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
catch (BrokenBarrierException e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
throw new IOException(new TestContainerLauncherImpl.ContainerException("Force fail CM"
));
}
/// <exception cref="System.Exception"/>
public virtual void TestNMProxyRetry()
{
containerManager.Start();
containerManager.SetBlockNewContainerRequests(false);
StartContainersRequest allRequests = Records.NewRecord <StartContainersRequest>();
ApplicationId appId = ApplicationId.NewInstance(1, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
Token nmToken = context.GetNMTokenSecretManager().CreateNMToken(attemptId, context
.GetNodeId(), user);
IPEndPoint address = conf.GetSocketAddr(YarnConfiguration.NmBindHost, YarnConfiguration
.NmAddress, YarnConfiguration.DefaultNmAddress, YarnConfiguration.DefaultNmPort);
Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> token = ConverterUtils.
ConvertFromYarn(nmToken, SecurityUtil.BuildTokenService(address));
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(user);
ugi.AddToken(token);
ContainerManagementProtocol proxy = NMProxy.CreateNMProxy <ContainerManagementProtocol
>(conf, ugi, YarnRPC.Create(conf), address);
retryCount = 0;
shouldThrowNMNotYetReadyException = false;
proxy.StartContainers(allRequests);
NUnit.Framework.Assert.AreEqual(5, retryCount);
retryCount = 0;
shouldThrowNMNotYetReadyException = false;
proxy.StopContainers(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <StopContainersRequest
>());
NUnit.Framework.Assert.AreEqual(5, retryCount);
retryCount = 0;
shouldThrowNMNotYetReadyException = false;
proxy.GetContainerStatuses(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetContainerStatusesRequest
>());
NUnit.Framework.Assert.AreEqual(5, retryCount);
retryCount = 0;
shouldThrowNMNotYetReadyException = true;
proxy.StartContainers(allRequests);
NUnit.Framework.Assert.AreEqual(5, retryCount);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public override StartContainersResponse StartContainers(StartContainersRequest requests
)
{
if (this._enclosing.retryCount < 5)
{
this._enclosing.retryCount++;
if (this._enclosing.shouldThrowNMNotYetReadyException)
{
// This causes super to throw an NMNotYetReadyException
this._enclosing.containerManager.SetBlockNewContainerRequests(true);
}
else
{
throw new ConnectException("start container exception");
}
}
else
{
// This stops super from throwing an NMNotYetReadyException
this._enclosing.containerManager.SetBlockNewContainerRequests(false);
}
return(base.StartContainers(requests));
}
public virtual void TestStartContainerFailureWithUnknownAuxService()
{
conf.SetStrings(YarnConfiguration.NmAuxServices, new string[] { "existService" });
conf.SetClass(string.Format(YarnConfiguration.NmAuxServiceFmt, "existService"), typeof(
TestAuxServices.ServiceA), typeof(Org.Apache.Hadoop.Service.Service));
containerManager.Start();
IList <StartContainerRequest> startRequest = new AList <StartContainerRequest>();
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
IDictionary <string, ByteBuffer> serviceData = new Dictionary <string, ByteBuffer>(
);
string serviceName = "non_exist_auxService";
serviceData[serviceName] = ByteBuffer.Wrap(Sharpen.Runtime.GetBytesForString(serviceName
));
containerLaunchContext.SetServiceData(serviceData);
ContainerId cId = CreateContainerId(0);
string user = "******";
Token containerToken = CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId
(), user, context.GetContainerTokenSecretManager());
StartContainerRequest request = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
// start containers
startRequest.AddItem(request);
StartContainersRequest requestList = StartContainersRequest.NewInstance(startRequest
);
StartContainersResponse response = containerManager.StartContainers(requestList);
NUnit.Framework.Assert.IsTrue(response.GetFailedRequests().Count == 1);
NUnit.Framework.Assert.IsTrue(response.GetSuccessfullyStartedContainers().Count ==
0);
NUnit.Framework.Assert.IsTrue(response.GetFailedRequests().Contains(cId));
NUnit.Framework.Assert.IsTrue(response.GetFailedRequests()[cId].GetMessage().Contains
("The auxService:" + serviceName + " does not exist"));
}
public virtual void Launch(ContainerRemoteLaunchEvent @event)
{
lock (this)
{
ContainerLauncherImpl.Log.Info("Launching " + this.taskAttemptID);
if (this.state == ContainerLauncherImpl.ContainerState.KilledBeforeLaunch)
{
this.state = ContainerLauncherImpl.ContainerState.Done;
this._enclosing.SendContainerLaunchFailedMsg(this.taskAttemptID, "Container was killed before it was launched"
);
return;
}
ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData proxy = null;
try
{
proxy = this._enclosing.GetCMProxy(this.containerMgrAddress, this.containerID);
// Construct the actual Container
ContainerLaunchContext containerLaunchContext = @event.GetContainerLaunchContext(
);
// Now launch the actual container
StartContainerRequest startRequest = StartContainerRequest.NewInstance(containerLaunchContext
, @event.GetContainerToken());
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(startRequest);
StartContainersRequest requestList = StartContainersRequest.NewInstance(list);
StartContainersResponse response = proxy.GetContainerManagementProtocol().StartContainers
(requestList);
if (response.GetFailedRequests() != null && response.GetFailedRequests().Contains
(this.containerID))
{
throw response.GetFailedRequests()[this.containerID].DeSerialize();
}
ByteBuffer portInfo = response.GetAllServicesMetaData()[ShuffleHandler.MapreduceShuffleServiceid
];
int port = -1;
if (portInfo != null)
{
port = ShuffleHandler.DeserializeMetaData(portInfo);
}
ContainerLauncherImpl.Log.Info("Shuffle port returned by ContainerManager for " +
this.taskAttemptID + " : " + port);
if (port < 0)
{
this.state = ContainerLauncherImpl.ContainerState.Failed;
throw new InvalidOperationException("Invalid shuffle port number " + port + " returned for "
+ this.taskAttemptID);
}
// after launching, send launched event to task attempt to move
// it from ASSIGNED to RUNNING state
this._enclosing.context.GetEventHandler().Handle(new TaskAttemptContainerLaunchedEvent
(this.taskAttemptID, port));
this.state = ContainerLauncherImpl.ContainerState.Running;
}
catch (Exception t)
{
string message = "Container launch failed for " + this.containerID + " : " + StringUtils
.StringifyException(t);
this.state = ContainerLauncherImpl.ContainerState.Failed;
this._enclosing.SendContainerLaunchFailedMsg(this.taskAttemptID, message);
}
finally
{
if (proxy != null)
{
this._enclosing.cmProxy.MayBeCloseProxy(proxy);
}
}
}
}
public virtual void TestMultipleContainersStopAndGetStatus()
{
containerManager.Start();
IList <StartContainerRequest> startRequest = new AList <StartContainerRequest>();
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
IList <ContainerId> containerIds = new AList <ContainerId>();
for (int i = 0; i < 10; i++)
{
ContainerId cId = CreateContainerId(i);
string user = null;
if ((i & 1) == 0)
{
// container with even id fail
user = "******";
}
else
{
user = "******";
}
Token containerToken = CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId
(), user, context.GetContainerTokenSecretManager());
StartContainerRequest request = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
startRequest.AddItem(request);
containerIds.AddItem(cId);
}
// start containers
StartContainersRequest requestList = StartContainersRequest.NewInstance(startRequest
);
containerManager.StartContainers(requestList);
// Get container statuses
GetContainerStatusesRequest statusRequest = GetContainerStatusesRequest.NewInstance
(containerIds);
GetContainerStatusesResponse statusResponse = containerManager.GetContainerStatuses
(statusRequest);
NUnit.Framework.Assert.AreEqual(5, statusResponse.GetContainerStatuses().Count);
foreach (ContainerStatus status in statusResponse.GetContainerStatuses())
{
// Containers with odd id should succeed
NUnit.Framework.Assert.AreEqual(1, status.GetContainerId().GetContainerId() & 1);
}
NUnit.Framework.Assert.AreEqual(5, statusResponse.GetFailedRequests().Count);
foreach (KeyValuePair <ContainerId, SerializedException> entry in statusResponse.GetFailedRequests
())
{
// Containers with even id should fail.
NUnit.Framework.Assert.AreEqual(0, entry.Key.GetContainerId() & 1);
NUnit.Framework.Assert.IsTrue(entry.Value.GetMessage().Contains("Reject this container"
));
}
// stop containers
StopContainersRequest stopRequest = StopContainersRequest.NewInstance(containerIds
);
StopContainersResponse stopResponse = containerManager.StopContainers(stopRequest
);
NUnit.Framework.Assert.AreEqual(5, stopResponse.GetSuccessfullyStoppedContainers(
).Count);
foreach (ContainerId id in stopResponse.GetSuccessfullyStoppedContainers())
{
// Containers with odd id should succeed.
NUnit.Framework.Assert.AreEqual(1, id.GetContainerId() & 1);
}
NUnit.Framework.Assert.AreEqual(5, stopResponse.GetFailedRequests().Count);
foreach (KeyValuePair <ContainerId, SerializedException> entry_1 in stopResponse.GetFailedRequests
())
{
// Containers with even id should fail.
NUnit.Framework.Assert.AreEqual(0, entry_1.Key.GetContainerId() & 1);
NUnit.Framework.Assert.IsTrue(entry_1.Value.GetMessage().Contains("Reject this container"
));
}
}
public _PrivilegedExceptionAction_152(TestNodeManagerReboot _enclosing, StartContainersRequest
allRequests)
{
this._enclosing = _enclosing;
this.allRequests = allRequests;
}
public virtual void TestContainerLaunchFromPreviousRM()
{
containerManager.Start();
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
ContainerId cId1 = CreateContainerId(0);
ContainerId cId2 = CreateContainerId(0);
containerLaunchContext.SetLocalResources(new Dictionary <string, LocalResource>());
// Construct the Container with Invalid RMIdentifier
StartContainerRequest startRequest1 = StartContainerRequest.NewInstance(containerLaunchContext
, CreateContainerToken(cId1, ResourceManagerConstants.RmInvalidIdentifier, context
.GetNodeId(), user, context.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(startRequest1);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
bool catchException = false;
try
{
StartContainersResponse response = containerManager.StartContainers(allRequests);
if (response.GetFailedRequests().Contains(cId1))
{
throw response.GetFailedRequests()[cId1].DeSerialize();
}
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
catchException = true;
NUnit.Framework.Assert.IsTrue(e.Message.Contains("Container " + cId1 + " rejected as it is allocated by a previous RM"
));
NUnit.Framework.Assert.IsTrue(Sharpen.Runtime.EqualsIgnoreCase(e.GetType().FullName
, typeof(InvalidContainerException).FullName));
}
// Verify that startContainer fail because of invalid container request
NUnit.Framework.Assert.IsTrue(catchException);
// Construct the Container with a RMIdentifier within current RM
StartContainerRequest startRequest2 = StartContainerRequest.NewInstance(containerLaunchContext
, CreateContainerToken(cId2, DummyRmIdentifier, context.GetNodeId(), user, context
.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list2 = new AList <StartContainerRequest>();
list.AddItem(startRequest2);
StartContainersRequest allRequests2 = StartContainersRequest.NewInstance(list2);
containerManager.StartContainers(allRequests2);
bool noException = true;
try
{
containerManager.StartContainers(allRequests2);
}
catch (YarnException)
{
noException = false;
}
// Verify that startContainer get no YarnException
NUnit.Framework.Assert.IsTrue(noException);
}
public virtual void TestLocalFilesCleanup()
{
// Real del service
delSrvc = new DeletionService(exec);
delSrvc.Init(conf);
containerManager = CreateContainerManager(delSrvc);
containerManager.Init(conf);
containerManager.Start();
// ////// Create the resources for the container
FilePath dir = new FilePath(tmpDir, "dir");
dir.Mkdirs();
FilePath file = new FilePath(dir, "file");
PrintWriter fileWriter = new PrintWriter(file);
fileWriter.Write("Hello World!");
fileWriter.Close();
// ////// Construct the Container-id
ContainerId cId = CreateContainerId(0);
ApplicationId appId = cId.GetApplicationAttemptId().GetApplicationId();
// ////// Construct the container-spec.
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
// containerLaunchContext.resources =
// new HashMap<CharSequence, LocalResource>();
URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(FileContext.GetLocalFSFileContext
().MakeQualified(new Path(file.GetAbsolutePath())));
LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();
rsrc_alpha.SetResource(resource_alpha);
rsrc_alpha.SetSize(-1);
rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
rsrc_alpha.SetType(LocalResourceType.File);
rsrc_alpha.SetTimestamp(file.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = rsrc_alpha;
containerLaunchContext.SetLocalResources(localResources);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context
.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
.Complete);
BaseContainerManagerTest.WaitForApplicationState(containerManager, cId.GetApplicationAttemptId
().GetApplicationId(), ApplicationState.Running);
// Now ascertain that the resources are localised correctly.
string appIDStr = ConverterUtils.ToString(appId);
string containerIDStr = ConverterUtils.ToString(cId);
FilePath userCacheDir = new FilePath(localDir, ContainerLocalizer.Usercache);
FilePath userDir = new FilePath(userCacheDir, user);
FilePath appCache = new FilePath(userDir, ContainerLocalizer.Appcache);
FilePath appDir = new FilePath(appCache, appIDStr);
FilePath containerDir = new FilePath(appDir, containerIDStr);
FilePath targetFile = new FilePath(containerDir, destinationFile);
FilePath sysDir = new FilePath(localDir, ResourceLocalizationService.NmPrivateDir
);
FilePath appSysDir = new FilePath(sysDir, appIDStr);
FilePath containerSysDir = new FilePath(appSysDir, containerIDStr);
// AppDir should still exist
NUnit.Framework.Assert.IsTrue("AppDir " + appDir.GetAbsolutePath() + " doesn't exist!!"
, appDir.Exists());
NUnit.Framework.Assert.IsTrue("AppSysDir " + appSysDir.GetAbsolutePath() + " doesn't exist!!"
, appSysDir.Exists());
foreach (FilePath f in new FilePath[] { containerDir, containerSysDir })
{
NUnit.Framework.Assert.IsFalse(f.GetAbsolutePath() + " exists!!", f.Exists());
}
NUnit.Framework.Assert.IsFalse(targetFile.GetAbsolutePath() + " exists!!", targetFile
.Exists());
// Simulate RM sending an AppFinish event.
containerManager.Handle(new CMgrCompletedAppsEvent(Arrays.AsList(new ApplicationId
[] { appId }), CMgrCompletedAppsEvent.Reason.OnShutdown));
BaseContainerManagerTest.WaitForApplicationState(containerManager, cId.GetApplicationAttemptId
().GetApplicationId(), ApplicationState.Finished);
// Now ascertain that the resources are localised correctly.
foreach (FilePath f_1 in new FilePath[] { appDir, containerDir, appSysDir, containerSysDir })
{
// Wait for deletion. Deletion can happen long after AppFinish because of
// the async DeletionService
int timeout = 0;
while (f_1.Exists() && timeout++ < 15)
{
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsFalse(f_1.GetAbsolutePath() + " exists!!", f_1.Exists());
}
// Wait for deletion
int timeout_1 = 0;
while (targetFile.Exists() && timeout_1++ < 15)
{
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsFalse(targetFile.GetAbsolutePath() + " exists!!", targetFile
.Exists());
}
public virtual void TestContainerLaunchAndStop()
{
containerManager.Start();
FilePath scriptFile = Shell.AppendScriptExtension(tmpDir, "scriptFile");
PrintWriter fileWriter = new PrintWriter(scriptFile);
FilePath processStartFile = new FilePath(tmpDir, "start_file.txt").GetAbsoluteFile
();
// ////// Construct the Container-id
ContainerId cId = CreateContainerId(0);
if (Shell.Windows)
{
fileWriter.WriteLine("@echo Hello World!> " + processStartFile);
fileWriter.WriteLine("@echo " + cId + ">> " + processStartFile);
fileWriter.WriteLine("@ping -n 100 127.0.0.1 >nul");
}
else
{
fileWriter.Write("\numask 0");
// So that start file is readable by the test
fileWriter.Write("\necho Hello World! > " + processStartFile);
fileWriter.Write("\necho $$ >> " + processStartFile);
fileWriter.Write("\nexec sleep 100");
}
fileWriter.Close();
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(scriptFile.GetAbsolutePath())));
LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();
rsrc_alpha.SetResource(resource_alpha);
rsrc_alpha.SetSize(-1);
rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
rsrc_alpha.SetType(LocalResourceType.File);
rsrc_alpha.SetTimestamp(scriptFile.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = rsrc_alpha;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile));
containerLaunchContext.SetCommands(commands);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context
.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
int timeoutSecs = 0;
while (!processStartFile.Exists() && timeoutSecs++ < 20)
{
Sharpen.Thread.Sleep(1000);
Log.Info("Waiting for process start-file to be created");
}
NUnit.Framework.Assert.IsTrue("ProcessStartFile doesn't exist!", processStartFile
.Exists());
// Now verify the contents of the file
BufferedReader reader = new BufferedReader(new FileReader(processStartFile));
NUnit.Framework.Assert.AreEqual("Hello World!", reader.ReadLine());
// Get the pid of the process
string pid = reader.ReadLine().Trim();
// No more lines
NUnit.Framework.Assert.AreEqual(null, reader.ReadLine());
// Now test the stop functionality.
// Assert that the process is alive
NUnit.Framework.Assert.IsTrue("Process is not alive!", DefaultContainerExecutor.ContainerIsAlive
(pid));
// Once more
NUnit.Framework.Assert.IsTrue("Process is not alive!", DefaultContainerExecutor.ContainerIsAlive
(pid));
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
StopContainersRequest stopRequest = StopContainersRequest.NewInstance(containerIds
);
containerManager.StopContainers(stopRequest);
BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
.Complete);
GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance(
containerIds);
ContainerStatus containerStatus = containerManager.GetContainerStatuses(gcsRequest
).GetContainerStatuses()[0];
int expectedExitCode = ContainerExitStatus.KilledByAppmaster;
NUnit.Framework.Assert.AreEqual(expectedExitCode, containerStatus.GetExitStatus()
);
// Assert that the process is not alive anymore
NUnit.Framework.Assert.IsFalse("Process is still alive!", DefaultContainerExecutor
.ContainerIsAlive(pid));
}
public virtual void TestContainerSetup()
{
containerManager.Start();
// ////// Create the resources for the container
FilePath dir = new FilePath(tmpDir, "dir");
dir.Mkdirs();
FilePath file = new FilePath(dir, "file");
PrintWriter fileWriter = new PrintWriter(file);
fileWriter.Write("Hello World!");
fileWriter.Close();
// ////// Construct the Container-id
ContainerId cId = CreateContainerId(0);
// ////// Construct the container-spec.
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(file.GetAbsolutePath())));
LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();
rsrc_alpha.SetResource(resource_alpha);
rsrc_alpha.SetSize(-1);
rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
rsrc_alpha.SetType(LocalResourceType.File);
rsrc_alpha.SetTimestamp(file.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = rsrc_alpha;
containerLaunchContext.SetLocalResources(localResources);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context
.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
.Complete);
// Now ascertain that the resources are localised correctly.
ApplicationId appId = cId.GetApplicationAttemptId().GetApplicationId();
string appIDStr = ConverterUtils.ToString(appId);
string containerIDStr = ConverterUtils.ToString(cId);
FilePath userCacheDir = new FilePath(localDir, ContainerLocalizer.Usercache);
FilePath userDir = new FilePath(userCacheDir, user);
FilePath appCache = new FilePath(userDir, ContainerLocalizer.Appcache);
FilePath appDir = new FilePath(appCache, appIDStr);
FilePath containerDir = new FilePath(appDir, containerIDStr);
FilePath targetFile = new FilePath(containerDir, destinationFile);
FilePath sysDir = new FilePath(localDir, ResourceLocalizationService.NmPrivateDir
);
FilePath appSysDir = new FilePath(sysDir, appIDStr);
FilePath containerSysDir = new FilePath(appSysDir, containerIDStr);
foreach (FilePath f in new FilePath[] { localDir, sysDir, userCacheDir, appDir, appSysDir
, containerDir, containerSysDir })
{
NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath() + " doesn't exist!!", f.Exists(
));
NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath() + " is not a directory!!", f.IsDirectory
());
}
NUnit.Framework.Assert.IsTrue(targetFile.GetAbsolutePath() + " doesn't exist!!",
targetFile.Exists());
// Now verify the contents of the file
BufferedReader reader = new BufferedReader(new FileReader(targetFile));
NUnit.Framework.Assert.AreEqual("Hello World!", reader.ReadLine());
NUnit.Framework.Assert.AreEqual(null, reader.ReadLine());
}
/// <exception cref="System.Exception"/>
private void Test(string rpcClass)
{
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.IpcRpcImpl, rpcClass);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestRPC.DummyContainerManager
(this), addr, conf, null, 1);
server.Start();
RPC.SetProtocolEngine(conf, typeof(ContainerManagementProtocolPB), typeof(ProtobufRpcEngine
));
ContainerManagementProtocol proxy = (ContainerManagementProtocol)rpc.GetProxy(typeof(
ContainerManagementProtocol), NetUtils.GetConnectAddress(server), conf);
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
ApplicationId applicationId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 0);
ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 100);
NodeId nodeId = NodeId.NewInstance("localhost", 1234);
Resource resource = Resource.NewInstance(1234, 2);
ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
containerId, "localhost", "user", resource, Runtime.CurrentTimeMillis() + 10000,
42, 42, Priority.NewInstance(0), 0);
Token containerToken = NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString
("password"), containerTokenIdentifier);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
proxy.StartContainers(allRequests);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(containerId);
GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance(
containerIds);
GetContainerStatusesResponse response = proxy.GetContainerStatuses(gcsRequest);
IList <ContainerStatus> statuses = response.GetContainerStatuses();
//test remote exception
bool exception = false;
try
{
StopContainersRequest stopRequest = recordFactory.NewRecordInstance <StopContainersRequest
>();
stopRequest.SetContainerIds(containerIds);
proxy.StopContainers(stopRequest);
}
catch (YarnException e)
{
exception = true;
NUnit.Framework.Assert.IsTrue(e.Message.Contains(ExceptionMsg));
NUnit.Framework.Assert.IsTrue(e.Message.Contains(ExceptionCause));
System.Console.Out.WriteLine("Test Exception is " + e.Message);
}
catch (Exception ex)
{
Sharpen.Runtime.PrintStackTrace(ex);
}
NUnit.Framework.Assert.IsTrue(exception);
server.Stop();
NUnit.Framework.Assert.IsNotNull(statuses[0]);
NUnit.Framework.Assert.AreEqual(ContainerState.Running, statuses[0].GetState());
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.Exception"/>
public virtual void TestClearLocalDirWhenNodeReboot()
{
nm = new TestNodeManagerReboot.MyNodeManager(this);
nm.Start();
ContainerManagementProtocol containerManager = nm.GetContainerManager();
// create files under fileCache
CreateFiles(nmLocalDir.GetAbsolutePath(), ContainerLocalizer.Filecache, 100);
localResourceDir.Mkdirs();
ContainerLaunchContext containerLaunchContext = Records.NewRecord <ContainerLaunchContext
>();
// Construct the Container-id
ContainerId cId = CreateContainerId();
URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(localResourceDir.GetAbsolutePath())));
LocalResource localResource = LocalResource.NewInstance(localResourceUri, LocalResourceType
.File, LocalResourceVisibility.Application, -1, localResourceDir.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = localResource;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = new AList <string>();
containerLaunchContext.SetCommands(commands);
NodeId nodeId = nm.GetNMContext().GetNodeId();
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, TestContainerManager.CreateContainerToken(cId, 0, nodeId, destinationFile, nm.
GetNMContext().GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.GetApplicationAttemptId
().ToString());
NMTokenIdentifier nmIdentifier = new NMTokenIdentifier(cId.GetApplicationAttemptId
(), nodeId, user, 123);
currentUser.AddTokenIdentifier(nmIdentifier);
currentUser.DoAs(new _PrivilegedExceptionAction_152(this, allRequests));
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds
);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= nm.GetNMContext().GetContainers()[request.GetContainerIds()[0]];
int MaxTries = 20;
int numTries = 0;
while (!container.GetContainerState().Equals(ContainerState.Done) && numTries <=
MaxTries)
{
try
{
Sharpen.Thread.Sleep(500);
}
catch (Exception)
{
}
// Do nothing
numTries++;
}
NUnit.Framework.Assert.AreEqual(ContainerState.Done, container.GetContainerState(
));
NUnit.Framework.Assert.IsTrue("The container should create a subDir named currentUser: "******"under localDir/usercache", NumOfLocalDirs(nmLocalDir.GetAbsolutePath(
), ContainerLocalizer.Usercache) > 0);
NUnit.Framework.Assert.IsTrue("There should be files or Dirs under nm_private when "
+ "container is launched", NumOfLocalDirs(nmLocalDir.GetAbsolutePath(), ResourceLocalizationService
.NmPrivateDir) > 0);
// restart the NodeManager
RestartNM(MaxTries);
CheckNumOfLocalDirs();
Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).Delete((string
)Matchers.IsNull(), Matchers.ArgThat(new TestNodeManagerReboot.PathInclude(this,
ResourceLocalizationService.NmPrivateDir + "_DEL_")));
Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).Delete((string
)Matchers.IsNull(), Matchers.ArgThat(new TestNodeManagerReboot.PathInclude(this,
ContainerLocalizer.Filecache + "_DEL_")));
Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).ScheduleFileDeletionTask
(Matchers.ArgThat(new TestNodeManagerReboot.FileDeletionInclude(this, user, null
, new string[] { destinationFile })));
Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).ScheduleFileDeletionTask
(Matchers.ArgThat(new TestNodeManagerReboot.FileDeletionInclude(this, null, ContainerLocalizer
.Usercache + "_DEL_", new string[] { })));
// restart the NodeManager again
// this time usercache directory should be empty
RestartNM(MaxTries);
CheckNumOfLocalDirs();
}
public virtual void TestContainerKillOnMemoryOverflow()
{
if (!ProcfsBasedProcessTree.IsAvailable())
{
return;
}
containerManager.Start();
FilePath scriptFile = new FilePath(tmpDir, "scriptFile.sh");
PrintWriter fileWriter = new PrintWriter(scriptFile);
FilePath processStartFile = new FilePath(tmpDir, "start_file.txt").GetAbsoluteFile
();
fileWriter.Write("\numask 0");
// So that start file is readable by the
// test.
fileWriter.Write("\necho Hello World! > " + processStartFile);
fileWriter.Write("\necho $$ >> " + processStartFile);
fileWriter.Write("\nsleep 15");
fileWriter.Close();
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
// ////// Construct the Container-id
ApplicationId appId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 1);
ContainerId cId = ContainerId.NewContainerId(appAttemptId, 0);
int port = 12345;
URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(scriptFile.GetAbsolutePath())));
LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();
rsrc_alpha.SetResource(resource_alpha);
rsrc_alpha.SetSize(-1);
rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
rsrc_alpha.SetType(LocalResourceType.File);
rsrc_alpha.SetTimestamp(scriptFile.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = rsrc_alpha;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = new AList <string>();
commands.AddItem("/bin/bash");
commands.AddItem(scriptFile.GetAbsolutePath());
containerLaunchContext.SetCommands(commands);
Resource r = BuilderUtils.NewResource(8 * 1024 * 1024, 1);
ContainerTokenIdentifier containerIdentifier = new ContainerTokenIdentifier(cId,
context.GetNodeId().ToString(), user, r, Runtime.CurrentTimeMillis() + 120000, 123
, DummyRmIdentifier, Priority.NewInstance(0), 0);
Token containerToken = BuilderUtils.NewContainerToken(context.GetNodeId(), containerManager
.GetContext().GetContainerTokenSecretManager().CreatePassword(containerIdentifier
), containerIdentifier);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
int timeoutSecs = 0;
while (!processStartFile.Exists() && timeoutSecs++ < 20)
{
Sharpen.Thread.Sleep(1000);
Log.Info("Waiting for process start-file to be created");
}
NUnit.Framework.Assert.IsTrue("ProcessStartFile doesn't exist!", processStartFile
.Exists());
// Now verify the contents of the file
BufferedReader reader = new BufferedReader(new FileReader(processStartFile));
NUnit.Framework.Assert.AreEqual("Hello World!", reader.ReadLine());
// Get the pid of the process
string pid = reader.ReadLine().Trim();
// No more lines
NUnit.Framework.Assert.AreEqual(null, reader.ReadLine());
BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
.Complete, 60);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance(
containerIds);
ContainerStatus containerStatus = containerManager.GetContainerStatuses(gcsRequest
).GetContainerStatuses()[0];
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.KilledExceededVmem, containerStatus
.GetExitStatus());
string expectedMsgPattern = "Container \\[pid=" + pid + ",containerID=" + cId + "\\] is running beyond virtual memory limits. Current usage: "
+ "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B physical memory used; " + "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B virtual memory used. "
+ "Killing container.\nDump of the process-tree for " + cId + " :\n";
Sharpen.Pattern pat = Sharpen.Pattern.Compile(expectedMsgPattern);
NUnit.Framework.Assert.AreEqual("Expected message pattern is: " + expectedMsgPattern
+ "\n\nObserved message is: " + containerStatus.GetDiagnostics(), true, pat.Matcher
(containerStatus.GetDiagnostics()).Find());
// Assert that the process is not alive anymore
NUnit.Framework.Assert.IsFalse("Process is still alive!", exec.SignalContainer(user
, pid, ContainerExecutor.Signal.Null));
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public override IDictionary <string, ByteBuffer> StartContainer(Container container
, ContainerLaunchContext containerLaunchContext)
{
// Do synchronization on StartedContainer to prevent race condition
// between startContainer and stopContainer only when startContainer is
// in progress for a given container.
NMClientImpl.StartedContainer startingContainer = CreateStartedContainer(container
);
lock (startingContainer)
{
AddStartingContainer(startingContainer);
IDictionary <string, ByteBuffer> allServiceResponse;
ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData proxy = null;
try
{
proxy = cmProxy.GetProxy(container.GetNodeId().ToString(), container.GetId());
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, container.GetContainerToken());
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
StartContainersResponse response = proxy.GetContainerManagementProtocol().StartContainers
(allRequests);
if (response.GetFailedRequests() != null && response.GetFailedRequests().Contains
(container.GetId()))
{
Exception t = response.GetFailedRequests()[container.GetId()].DeSerialize();
ParseAndThrowException(t);
}
allServiceResponse = response.GetAllServicesMetaData();
startingContainer.state = ContainerState.Running;
}
catch (YarnException e)
{
startingContainer.state = ContainerState.Complete;
// Remove the started container if it failed to start
RemoveStartedContainer(startingContainer);
throw;
}
catch (IOException e)
{
startingContainer.state = ContainerState.Complete;
RemoveStartedContainer(startingContainer);
throw;
}
catch (Exception t)
{
startingContainer.state = ContainerState.Complete;
RemoveStartedContainer(startingContainer);
throw RPCUtil.GetRemoteException(t);
}
finally
{
if (proxy != null)
{
cmProxy.MayBeCloseProxy(proxy);
}
}
return(allServiceResponse);
}
}
/// <exception cref="System.Exception"/>
public StartContainersResponse Run()
{
return(cm.StartContainers(StartContainersRequest.NewInstance(scReqList)));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private void TestContainerLaunchAndExit(int exitCode)
{
FilePath scriptFile = Shell.AppendScriptExtension(tmpDir, "scriptFile");
PrintWriter fileWriter = new PrintWriter(scriptFile);
FilePath processStartFile = new FilePath(tmpDir, "start_file.txt").GetAbsoluteFile
();
// ////// Construct the Container-id
ContainerId cId = CreateContainerId(0);
if (Shell.Windows)
{
fileWriter.WriteLine("@echo Hello World!> " + processStartFile);
fileWriter.WriteLine("@echo " + cId + ">> " + processStartFile);
if (exitCode != 0)
{
fileWriter.WriteLine("@exit " + exitCode);
}
}
else
{
fileWriter.Write("\numask 0");
// So that start file is readable by the test
fileWriter.Write("\necho Hello World! > " + processStartFile);
fileWriter.Write("\necho $$ >> " + processStartFile);
// Have script throw an exit code at the end
if (exitCode != 0)
{
fileWriter.Write("\nexit " + exitCode);
}
}
fileWriter.Close();
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(scriptFile.GetAbsolutePath())));
LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();
rsrc_alpha.SetResource(resource_alpha);
rsrc_alpha.SetSize(-1);
rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
rsrc_alpha.SetType(LocalResourceType.File);
rsrc_alpha.SetTimestamp(scriptFile.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = rsrc_alpha;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile));
containerLaunchContext.SetCommands(commands);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context
.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
.Complete);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance(
containerIds);
ContainerStatus containerStatus = containerManager.GetContainerStatuses(gcsRequest
).GetContainerStatuses()[0];
// Verify exit status matches exit state of script
NUnit.Framework.Assert.AreEqual(exitCode, containerStatus.GetExitStatus());
}
