/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual StopContainersResponse StopContainers(StopContainersRequest request
)
{
lock (this)
{
foreach (ContainerId containerID in request.GetContainerIds())
{
string applicationId = containerID.GetApplicationAttemptId().GetApplicationId().GetId
().ToString();
// Mark the container as COMPLETE
IList <Container> applicationContainers = containers[containerID.GetApplicationAttemptId
().GetApplicationId()];
foreach (Container c in applicationContainers)
{
if (c.GetId().CompareTo(containerID) == 0)
{
ContainerStatus containerStatus = containerStatusMap[c];
containerStatus.SetState(ContainerState.Complete);
containerStatusMap[c] = containerStatus;
}
}
// Send a heartbeat
try
{
Heartbeat();
}
catch (IOException ioe)
{
throw RPCUtil.GetRemoteException(ioe);
}
// Remove container and update status
int ctr = 0;
Container container = null;
for (IEnumerator <Container> i = applicationContainers.GetEnumerator(); i.HasNext(
);)
{
container = i.Next();
if (container.GetId().CompareTo(containerID) == 0)
{
i.Remove();
++ctr;
}
}
if (ctr != 1)
{
throw new InvalidOperationException("Container " + containerID + " stopped " + ctr
+ " times!");
}
Resources.AddTo(available, container.GetResource());
Resources.SubtractFrom(used, container.GetResource());
if (Log.IsDebugEnabled())
{
Log.Debug("stopContainer:" + " node=" + containerManagerAddress + " application="
+ applicationId + " container=" + containerID + " available=" + available + " used="
+ used);
}
}
return(StopContainersResponse.NewInstance(null, null));
}
}
/// <exception cref="System.IO.IOException"/>
public virtual StopContainersResponse StopContainers(StopContainersRequest request
)
{
Exception e = new Exception("Dummy function", new Exception("Dummy function cause"
));
throw new IOException(e);
}
/// <exception cref="System.Exception"/>
private void StopContainer(YarnRPC rpc, Token nmToken, IList <ContainerId> containerId
, ApplicationAttemptId appAttemptId, NodeId nodeId)
{
StopContainersRequest request = StopContainersRequest.NewInstance(containerId);
ContainerManagementProtocol proxy = null;
try
{
proxy = GetContainerManagementProtocolProxy(rpc, nmToken, nodeId, appAttemptId.ToString
());
StopContainersResponse response = proxy.StopContainers(request);
if (response.GetFailedRequests() != null && response.GetFailedRequests().Contains
(containerId))
{
ParseAndThrowException(response.GetFailedRequests()[containerId].DeSerialize());
}
}
catch (Exception)
{
if (proxy != null)
{
rpc.StopProxy(proxy, conf);
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private void StopContainerInternal(ContainerId containerId, NodeId nodeId)
{
ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData proxy = null;
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(containerId);
try
{
proxy = cmProxy.GetProxy(nodeId.ToString(), containerId);
StopContainersResponse response = proxy.GetContainerManagementProtocol().StopContainers
(StopContainersRequest.NewInstance(containerIds));
if (response.GetFailedRequests() != null && response.GetFailedRequests().Contains
(containerId))
{
Exception t = response.GetFailedRequests()[containerId].DeSerialize();
ParseAndThrowException(t);
}
}
finally
{
if (proxy != null)
{
cmProxy.MayBeCloseProxy(proxy);
}
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual StopContainersResponse StopContainers(StopContainersRequest request
)
{
Exception e = new Exception(TestRPC.ExceptionMsg, new Exception(TestRPC.ExceptionCause
));
throw new YarnException(e);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public override StopContainersResponse StopContainers(StopContainersRequest requests
)
{
if (this._enclosing.retryCount < 5)
{
this._enclosing.retryCount++;
throw new ConnectException("stop container exception");
}
return(base.StopContainers(requests));
}
public virtual void Kill()
{
lock (this)
{
if (this.state == ContainerLauncherImpl.ContainerState.Prep)
{
this.state = ContainerLauncherImpl.ContainerState.KilledBeforeLaunch;
}
else
{
if (!this.IsCompletelyDone())
{
ContainerLauncherImpl.Log.Info("KILLING " + this.taskAttemptID);
ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData proxy = null;
try
{
proxy = this._enclosing.GetCMProxy(this.containerMgrAddress, this.containerID);
// kill the remote container if already launched
IList <ContainerId> ids = new AList <ContainerId>();
ids.AddItem(this.containerID);
StopContainersRequest request = StopContainersRequest.NewInstance(ids);
StopContainersResponse response = proxy.GetContainerManagementProtocol().StopContainers
(request);
if (response.GetFailedRequests() != null && response.GetFailedRequests().Contains
(this.containerID))
{
throw response.GetFailedRequests()[this.containerID].DeSerialize();
}
}
catch (Exception t)
{
// ignore the cleanup failure
string message = "cleanup failed for container " + this.containerID + " : " + StringUtils
.StringifyException(t);
this._enclosing.context.GetEventHandler().Handle(new TaskAttemptDiagnosticsUpdateEvent
(this.taskAttemptID, message));
ContainerLauncherImpl.Log.Warn(message);
}
finally
{
if (proxy != null)
{
this._enclosing.cmProxy.MayBeCloseProxy(proxy);
}
}
this.state = ContainerLauncherImpl.ContainerState.Done;
}
}
// after killing, send killed event to task attempt
this._enclosing.context.GetEventHandler().Handle(new TaskAttemptEvent(this.taskAttemptID
, TaskAttemptEventType.TaContainerCleaned));
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual StopContainersResponse StopContainers(StopContainersRequest requests
)
{
YarnServiceProtos.StopContainersRequestProto requestProto = ((StopContainersRequestPBImpl
)requests).GetProto();
try
{
return(new StopContainersResponsePBImpl(proxy.StopContainers(null, requestProto)));
}
catch (ServiceException e)
{
RPCUtil.UnwrapAndThrowException(e);
return(null);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private void Cleanup()
{
Connect();
ContainerId containerId = masterContainer.GetId();
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(containerId);
StopContainersRequest stopRequest = StopContainersRequest.NewInstance(containerIds
);
StopContainersResponse response = containerMgrProxy.StopContainers(stopRequest);
if (response.GetFailedRequests() != null && response.GetFailedRequests().Contains
(containerId))
{
Exception t = response.GetFailedRequests()[containerId].DeSerialize();
ParseAndThrowException(t);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual void FinishTask(Task task)
{
lock (this)
{
ICollection <Task> tasks = this.tasks[task.GetPriority()];
if (!tasks.Remove(task))
{
throw new InvalidOperationException("Finishing unknown task " + task.GetTaskId()
+ " from application " + applicationId);
}
NodeManager nodeManager = task.GetNodeManager();
ContainerId containerId = task.GetContainerId();
task.Stop();
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(containerId);
StopContainersRequest stopRequest = StopContainersRequest.NewInstance(containerIds
);
nodeManager.StopContainers(stopRequest);
Resources.SubtractFrom(used, requestSpec[task.GetPriority()]);
Log.Info("Finished task " + task.GetTaskId() + " of application " + applicationId
+ " on node " + nodeManager.GetHostName() + ", currently using " + used + " resources"
);
}
}
public virtual void TestContainerLaunchAndStop()
{
containerManager.Start();
FilePath scriptFile = Shell.AppendScriptExtension(tmpDir, "scriptFile");
PrintWriter fileWriter = new PrintWriter(scriptFile);
FilePath processStartFile = new FilePath(tmpDir, "start_file.txt").GetAbsoluteFile
();
// ////// Construct the Container-id
ContainerId cId = CreateContainerId(0);
if (Shell.Windows)
{
fileWriter.WriteLine("@echo Hello World!> " + processStartFile);
fileWriter.WriteLine("@echo " + cId + ">> " + processStartFile);
fileWriter.WriteLine("@ping -n 100 127.0.0.1 >nul");
}
else
{
fileWriter.Write("\numask 0");
// So that start file is readable by the test
fileWriter.Write("\necho Hello World! > " + processStartFile);
fileWriter.Write("\necho $$ >> " + processStartFile);
fileWriter.Write("\nexec sleep 100");
}
fileWriter.Close();
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(scriptFile.GetAbsolutePath())));
LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();
rsrc_alpha.SetResource(resource_alpha);
rsrc_alpha.SetSize(-1);
rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
rsrc_alpha.SetType(LocalResourceType.File);
rsrc_alpha.SetTimestamp(scriptFile.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = rsrc_alpha;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile));
containerLaunchContext.SetCommands(commands);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context
.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
int timeoutSecs = 0;
while (!processStartFile.Exists() && timeoutSecs++ < 20)
{
Sharpen.Thread.Sleep(1000);
Log.Info("Waiting for process start-file to be created");
}
NUnit.Framework.Assert.IsTrue("ProcessStartFile doesn't exist!", processStartFile
.Exists());
// Now verify the contents of the file
BufferedReader reader = new BufferedReader(new FileReader(processStartFile));
NUnit.Framework.Assert.AreEqual("Hello World!", reader.ReadLine());
// Get the pid of the process
string pid = reader.ReadLine().Trim();
// No more lines
NUnit.Framework.Assert.AreEqual(null, reader.ReadLine());
// Now test the stop functionality.
// Assert that the process is alive
NUnit.Framework.Assert.IsTrue("Process is not alive!", DefaultContainerExecutor.ContainerIsAlive
(pid));
// Once more
NUnit.Framework.Assert.IsTrue("Process is not alive!", DefaultContainerExecutor.ContainerIsAlive
(pid));
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
StopContainersRequest stopRequest = StopContainersRequest.NewInstance(containerIds
);
containerManager.StopContainers(stopRequest);
BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
.Complete);
GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance(
containerIds);
ContainerStatus containerStatus = containerManager.GetContainerStatuses(gcsRequest
).GetContainerStatuses()[0];
int expectedExitCode = ContainerExitStatus.KilledByAppmaster;
NUnit.Framework.Assert.AreEqual(expectedExitCode, containerStatus.GetExitStatus()
);
// Assert that the process is not alive anymore
NUnit.Framework.Assert.IsFalse("Process is still alive!", DefaultContainerExecutor
.ContainerIsAlive(pid));
}
/// <exception cref="System.IO.IOException"/>
public virtual StopContainersResponse StopContainers(StopContainersRequest request
)
{
return(null);
}
/// <exception cref="System.Exception"/>
private void Test(string rpcClass)
{
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.IpcRpcImpl, rpcClass);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestRPC.DummyContainerManager
(this), addr, conf, null, 1);
server.Start();
RPC.SetProtocolEngine(conf, typeof(ContainerManagementProtocolPB), typeof(ProtobufRpcEngine
));
ContainerManagementProtocol proxy = (ContainerManagementProtocol)rpc.GetProxy(typeof(
ContainerManagementProtocol), NetUtils.GetConnectAddress(server), conf);
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
ApplicationId applicationId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 0);
ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 100);
NodeId nodeId = NodeId.NewInstance("localhost", 1234);
Resource resource = Resource.NewInstance(1234, 2);
ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
containerId, "localhost", "user", resource, Runtime.CurrentTimeMillis() + 10000,
42, 42, Priority.NewInstance(0), 0);
Token containerToken = NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString
("password"), containerTokenIdentifier);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
proxy.StartContainers(allRequests);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(containerId);
GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance(
containerIds);
GetContainerStatusesResponse response = proxy.GetContainerStatuses(gcsRequest);
IList <ContainerStatus> statuses = response.GetContainerStatuses();
//test remote exception
bool exception = false;
try
{
StopContainersRequest stopRequest = recordFactory.NewRecordInstance <StopContainersRequest
>();
stopRequest.SetContainerIds(containerIds);
proxy.StopContainers(stopRequest);
}
catch (YarnException e)
{
exception = true;
NUnit.Framework.Assert.IsTrue(e.Message.Contains(ExceptionMsg));
NUnit.Framework.Assert.IsTrue(e.Message.Contains(ExceptionCause));
System.Console.Out.WriteLine("Test Exception is " + e.Message);
}
catch (Exception ex)
{
Sharpen.Runtime.PrintStackTrace(ex);
}
NUnit.Framework.Assert.IsTrue(exception);
server.Stop();
NUnit.Framework.Assert.IsNotNull(statuses[0]);
NUnit.Framework.Assert.AreEqual(ContainerState.Running, statuses[0].GetState());
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public StopContainersResponse StopContainers(StopContainersRequest request)
{
return(StopContainersResponse.NewInstance(null, null));
}
public virtual void TestMultipleContainersStopAndGetStatus()
{
containerManager.Start();
IList <StartContainerRequest> startRequest = new AList <StartContainerRequest>();
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
IList <ContainerId> containerIds = new AList <ContainerId>();
for (int i = 0; i < 10; i++)
{
ContainerId cId = CreateContainerId(i);
string user = null;
if ((i & 1) == 0)
{
// container with even id fail
user = "******";
}
else
{
user = "******";
}
Token containerToken = CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId
(), user, context.GetContainerTokenSecretManager());
StartContainerRequest request = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
startRequest.AddItem(request);
containerIds.AddItem(cId);
}
// start containers
StartContainersRequest requestList = StartContainersRequest.NewInstance(startRequest
);
containerManager.StartContainers(requestList);
// Get container statuses
GetContainerStatusesRequest statusRequest = GetContainerStatusesRequest.NewInstance
(containerIds);
GetContainerStatusesResponse statusResponse = containerManager.GetContainerStatuses
(statusRequest);
NUnit.Framework.Assert.AreEqual(5, statusResponse.GetContainerStatuses().Count);
foreach (ContainerStatus status in statusResponse.GetContainerStatuses())
{
// Containers with odd id should succeed
NUnit.Framework.Assert.AreEqual(1, status.GetContainerId().GetContainerId() & 1);
}
NUnit.Framework.Assert.AreEqual(5, statusResponse.GetFailedRequests().Count);
foreach (KeyValuePair <ContainerId, SerializedException> entry in statusResponse.GetFailedRequests
())
{
// Containers with even id should fail.
NUnit.Framework.Assert.AreEqual(0, entry.Key.GetContainerId() & 1);
NUnit.Framework.Assert.IsTrue(entry.Value.GetMessage().Contains("Reject this container"
));
}
// stop containers
StopContainersRequest stopRequest = StopContainersRequest.NewInstance(containerIds
);
StopContainersResponse stopResponse = containerManager.StopContainers(stopRequest
);
NUnit.Framework.Assert.AreEqual(5, stopResponse.GetSuccessfullyStoppedContainers(
).Count);
foreach (ContainerId id in stopResponse.GetSuccessfullyStoppedContainers())
{
// Containers with odd id should succeed.
NUnit.Framework.Assert.AreEqual(1, id.GetContainerId() & 1);
}
NUnit.Framework.Assert.AreEqual(5, stopResponse.GetFailedRequests().Count);
foreach (KeyValuePair <ContainerId, SerializedException> entry_1 in stopResponse.GetFailedRequests
())
{
// Containers with even id should fail.
NUnit.Framework.Assert.AreEqual(0, entry_1.Key.GetContainerId() & 1);
NUnit.Framework.Assert.IsTrue(entry_1.Value.GetMessage().Contains("Reject this container"
));
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public StopContainersResponse StopContainers(StopContainersRequest request)
{
Log.Info("Container cleaned up by MyContainerManager");
cleanedup = true;
return(null);
}
public virtual void TestSuccessfulContainerLaunch()
{
FileContext localFS = FileContext.GetLocalFSFileContext();
localFS.Delete(new Path(localDir.GetAbsolutePath()), true);
localFS.Delete(new Path(localLogDir.GetAbsolutePath()), true);
localFS.Delete(new Path(remoteLogDir.GetAbsolutePath()), true);
localDir.Mkdir();
localLogDir.Mkdir();
remoteLogDir.Mkdir();
YarnConfiguration conf = new YarnConfiguration();
Context context = new _NMContext_84(new NMContainerTokenSecretManager(conf), new
NMTokenSecretManagerInNM(), null, null, new NMNullStateStoreService());
conf.Set(YarnConfiguration.NmLocalDirs, localDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmLogDirs, localLogDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmRemoteAppLogDir, remoteLogDir.GetAbsolutePath());
ContainerExecutor exec = new DefaultContainerExecutor();
exec.SetConf(conf);
DeletionService del = new DeletionService(exec);
Dispatcher dispatcher = new AsyncDispatcher();
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
NodeManagerMetrics metrics = NodeManagerMetrics.Create();
NodeStatusUpdater nodeStatusUpdater = new _NodeStatusUpdaterImpl_106(context, dispatcher
, healthChecker, metrics);
// Don't start any updating thread.
DummyContainerManager containerManager = new DummyContainerManager(context, exec,
del, nodeStatusUpdater, metrics, new ApplicationACLsManager(conf), dirsHandler);
nodeStatusUpdater.Init(conf);
((NodeManager.NMContext)context).SetContainerManager(containerManager);
nodeStatusUpdater.Start();
containerManager.Init(conf);
containerManager.Start();
ContainerLaunchContext launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
ApplicationId applicationId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 0);
ContainerId cID = ContainerId.NewContainerId(applicationAttemptId, 0);
string user = "******";
StartContainerRequest scRequest = StartContainerRequest.NewInstance(launchContext
, TestContainerManager.CreateContainerToken(cID, SimulatedRmIdentifier, context.
GetNodeId(), user, context.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
BaseContainerManagerTest.WaitForContainerState(containerManager, cID, ContainerState
.Running);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cID);
StopContainersRequest stopRequest = StopContainersRequest.NewInstance(containerIds
);
containerManager.StopContainers(stopRequest);
BaseContainerManagerTest.WaitForContainerState(containerManager, cID, ContainerState
.Complete);
containerManager.Stop();
}
