public virtual void Handle(ContainersLauncherEvent @event)
{
// TODO: ContainersLauncher launches containers one by one!!
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= @event.GetContainer();
ContainerId containerId = container.GetContainerId();
switch (@event.GetType())
{
case ContainersLauncherEventType.LaunchContainer:
{
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = context.GetApplications()[containerId.GetApplicationAttemptId().GetApplicationId
()];
ContainerLaunch launch = new ContainerLaunch(context, GetConfig(), dispatcher, exec
, app, @event.GetContainer(), dirsHandler, containerManager);
containerLauncher.Submit(launch);
running[containerId] = launch;
break;
}
case ContainersLauncherEventType.RecoverContainer:
{
app = context.GetApplications()[containerId.GetApplicationAttemptId().GetApplicationId
()];
launch = new RecoveredContainerLaunch(context, GetConfig(), dispatcher, exec, app
, @event.GetContainer(), dirsHandler, containerManager);
containerLauncher.Submit(launch);
running[containerId] = launch;
break;
}
case ContainersLauncherEventType.CleanupContainer:
{
ContainerLaunch launcher = Sharpen.Collections.Remove(running, containerId);
if (launcher == null)
{
// Container not launched. So nothing needs to be done.
return;
}
// Cleanup a container whether it is running/killed/completed, so that
// no sub-processes are alive.
try
{
launcher.CleanupContainer();
}
catch (IOException)
{
Log.Warn("Got exception while cleaning container " + containerId + ". Ignoring.");
}
break;
}
}
}
public virtual void TestForCorruptedAggregatedLogs()
{
Configuration conf = new Configuration();
FilePath workDir = new FilePath(testWorkDir, "testReadAcontainerLogs1");
Path remoteAppLogFile = new Path(workDir.GetAbsolutePath(), "aggregatedLogFile");
Path srcFileRoot = new Path(workDir.GetAbsolutePath(), "srcFiles");
ContainerId testContainerId = TestContainerId.NewContainerId(1, 1, 1, 1);
Path t = new Path(srcFileRoot, testContainerId.GetApplicationAttemptId().GetApplicationId
().ToString());
Path srcFilePath = new Path(t, testContainerId.ToString());
long numChars = 950000;
WriteSrcFileAndALog(srcFilePath, "stdout", numChars, remoteAppLogFile, srcFileRoot
, testContainerId);
AggregatedLogFormat.LogReader logReader = new AggregatedLogFormat.LogReader(conf,
remoteAppLogFile);
AggregatedLogFormat.LogKey rLogKey = new AggregatedLogFormat.LogKey();
DataInputStream dis = logReader.Next(rLogKey);
TextWriter writer = new StringWriter();
try
{
AggregatedLogFormat.LogReader.ReadAcontainerLogs(dis, writer);
}
catch (Exception e)
{
if (e.ToString().Contains("NumberFormatException"))
{
NUnit.Framework.Assert.Fail("Aggregated logs are corrupted.");
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestContainerLaunchNullImage()
{
string appSubmitter = "nobody";
string appId = "APP_ID";
string containerId = "CONTAINER_ID";
string testImage = string.Empty;
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>(Org.Mockito.Mockito.ReturnsDeepStubs);
ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(Org.Mockito.Mockito.ReturnsDeepStubs
);
ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
();
Dictionary <string, string> env = new Dictionary <string, string>();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
Org.Mockito.Mockito.When(cId.GetApplicationAttemptId().GetApplicationId().ToString
()).ThenReturn(appId);
Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId);
Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env);
env[YarnConfiguration.NmDockerContainerExecutorImageName] = testImage;
dockerContainerExecutor.GetConf().Set(YarnConfiguration.NmDockerContainerExecutorImageName
, testImage);
Path scriptPath = new Path("file:///bin/echo");
Path tokensPath = new Path("file:///dev/null");
Path pidFile = new Path(workDir, "pid.txt");
dockerContainerExecutor.ActivateContainer(cId, pidFile);
dockerContainerExecutor.LaunchContainer(container, scriptPath, tokensPath, appSubmitter
, appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs());
}
public virtual void TestLogDirWithDriveLetter()
{
//To verify that logs paths which include drive letters (Windows)
//do not lose their drive letter specification
LocalDirsHandlerService localDirs = Org.Mockito.Mockito.Mock <LocalDirsHandlerService
>();
IList <string> logDirs = new AList <string>();
logDirs.AddItem("F:/nmlogs");
Org.Mockito.Mockito.When(localDirs.GetLogDirsForRead()).ThenReturn(logDirs);
ApplicationIdPBImpl appId = Org.Mockito.Mockito.Mock <ApplicationIdPBImpl>();
Org.Mockito.Mockito.When(appId.ToString()).ThenReturn("app_id_1");
ApplicationAttemptIdPBImpl appAttemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptIdPBImpl
>();
Org.Mockito.Mockito.When(appAttemptId.GetApplicationId()).ThenReturn(appId);
ContainerId containerId = Org.Mockito.Mockito.Mock <ContainerIdPBImpl>();
Org.Mockito.Mockito.When(containerId.GetApplicationAttemptId()).ThenReturn(appAttemptId
);
IList <FilePath> logDirFiles = ContainerLogsUtils.GetContainerLogDirs(containerId,
localDirs);
NUnit.Framework.Assert.IsTrue("logDir lost drive letter " + logDirFiles[0], logDirFiles
[0].ToString().IndexOf("F:" + FilePath.separator + "nmlogs") > -1);
}
/// <exception cref="System.IO.IOException"/>
public static FileInputStream OpenLogFileForRead(string containerIdStr, FilePath
logFile, Context context)
{
ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr);
ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId
();
string user = context.GetApplications()[applicationId].GetUser();
try
{
return(SecureIOUtils.OpenForRead(logFile, user, null));
}
catch (IOException e)
{
if (e.Message.Contains("did not match expected owner '" + user + "'"))
{
Log.Error("Exception reading log file " + logFile.GetAbsolutePath(), e);
throw new IOException("Exception reading log file. Application submitted by '" +
user + "' doesn't own requested log file : " + logFile.GetName(), e);
}
else
{
throw new IOException("Exception reading log file. It might be because log " + "file was aggregated : "
+ logFile.GetName(), e);
}
}
}
// These NMContainerStatus are sent on NM registration and used by YARN only.
/// <exception cref="System.IO.IOException"/>
private IList <NMContainerStatus> GetNMContainerStatuses()
{
IList <NMContainerStatus> containerStatuses = new AList <NMContainerStatus>();
foreach (Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container in this.context.GetContainers().Values)
{
ContainerId containerId = container.GetContainerId();
ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId
();
if (!this.context.GetApplications().Contains(applicationId))
{
Sharpen.Collections.Remove(context.GetContainers(), containerId);
continue;
}
NMContainerStatus status = container.GetNMContainerStatus();
containerStatuses.AddItem(status);
if (status.GetContainerState() == ContainerState.Complete)
{
// Adding to finished containers cache. Cache will keep it around at
// least for #durationToTrackStoppedContainers duration. In the
// subsequent call to stop container it will get removed from cache.
AddCompletedContainer(containerId);
}
}
Log.Info("Sending out " + containerStatuses.Count + " NM container statuses: " +
containerStatuses);
return(containerStatuses);
}
/// <exception cref="System.IO.IOException"/>
private int RunAndBlock(ContainerId cId, IDictionary <string, string> launchCtxEnv
, params string[] cmd)
{
string appId = "APP_" + Runtime.CurrentTimeMillis();
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
Org.Mockito.Mockito.When(cId.GetApplicationAttemptId().GetApplicationId().ToString
()).ThenReturn(appId);
Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(launchCtxEnv);
string script = WriteScriptFile(launchCtxEnv, cmd);
Path scriptPath = new Path(script);
Path tokensPath = new Path("/dev/null");
Path workDir = new Path(workSpace.GetAbsolutePath());
Path pidFile = new Path(workDir, "pid.txt");
exec.ActivateContainer(cId, pidFile);
return(exec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter, appId
, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs()));
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public StartContainersResponse StartContainers(StartContainersRequest requests)
{
StartContainerRequest request = requests.GetStartContainerRequests()[0];
Log.Info("Container started by MyContainerManager: " + request);
launched = true;
IDictionary <string, string> env = request.GetContainerLaunchContext().GetEnvironment
();
Token containerToken = request.GetContainerToken();
ContainerTokenIdentifier tokenId = null;
try
{
tokenId = BuilderUtils.NewContainerTokenIdentifier(containerToken);
}
catch (IOException e)
{
throw RPCUtil.GetRemoteException(e);
}
ContainerId containerId = tokenId.GetContainerID();
containerIdAtContainerManager = containerId.ToString();
attemptIdAtContainerManager = containerId.GetApplicationAttemptId().ToString();
nmHostAtContainerManager = tokenId.GetNmHostAddress();
submitTimeAtContainerManager = long.Parse(env[ApplicationConstants.AppSubmitTimeEnv
]);
maxAppAttempts = System.Convert.ToInt32(env[ApplicationConstants.MaxAppAttemptsEnv
]);
return(StartContainersResponse.NewInstance(new Dictionary <string, ByteBuffer>(),
new AList <ContainerId>(), new Dictionary <ContainerId, SerializedException>()));
}
/// <exception cref="System.IO.IOException"/>
public virtual ContainerReport GetContainer(ContainerId containerId)
{
ApplicationReport app = GetApplication(containerId.GetApplicationAttemptId().GetApplicationId
());
return(ConvertToContainerReport(historyStore.GetContainer(containerId), app == null
? null : app.GetUser()));
}
/// <exception cref="System.IO.IOException"/>
public virtual ContainerHistoryData GetContainer(ContainerId containerId)
{
FileSystemApplicationHistoryStore.HistoryFileReader hfReader = GetHistoryFileReader
(containerId.GetApplicationAttemptId().GetApplicationId());
try
{
bool readStartData = false;
bool readFinishData = false;
ContainerHistoryData historyData = ContainerHistoryData.NewInstance(containerId,
null, null, null, long.MinValue, long.MaxValue, null, int.MaxValue, null);
while ((!readStartData || !readFinishData) && hfReader.HasNext())
{
FileSystemApplicationHistoryStore.HistoryFileReader.Entry entry = hfReader.Next();
if (entry.key.id.Equals(containerId.ToString()))
{
if (entry.key.suffix.Equals(StartDataSuffix))
{
ContainerStartData startData = ParseContainerStartData(entry.value);
MergeContainerHistoryData(historyData, startData);
readStartData = true;
}
else
{
if (entry.key.suffix.Equals(FinishDataSuffix))
{
ContainerFinishData finishData = ParseContainerFinishData(entry.value);
MergeContainerHistoryData(historyData, finishData);
readFinishData = true;
}
}
}
}
if (!readStartData && !readFinishData)
{
return(null);
}
if (!readStartData)
{
Log.Warn("Start information is missing for container " + containerId);
}
if (!readFinishData)
{
Log.Warn("Finish information is missing for container " + containerId);
}
Log.Info("Completed reading history information of container " + containerId);
return(historyData);
}
catch (IOException e)
{
Log.Error("Error when reading history file of container " + containerId, e);
throw;
}
finally
{
hfReader.Close();
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual StartContainersResponse StartContainers(StartContainersRequest requests
)
{
lock (this)
{
foreach (StartContainerRequest request in requests.GetStartContainerRequests())
{
Token containerToken = request.GetContainerToken();
ContainerTokenIdentifier tokenId = null;
try
{
tokenId = BuilderUtils.NewContainerTokenIdentifier(containerToken);
}
catch (IOException e)
{
throw RPCUtil.GetRemoteException(e);
}
ContainerId containerID = tokenId.GetContainerID();
ApplicationId applicationId = containerID.GetApplicationAttemptId().GetApplicationId
();
IList <Container> applicationContainers = containers[applicationId];
if (applicationContainers == null)
{
applicationContainers = new AList <Container>();
containers[applicationId] = applicationContainers;
}
// Sanity check
foreach (Container container in applicationContainers)
{
if (container.GetId().CompareTo(containerID) == 0)
{
throw new InvalidOperationException("Container " + containerID + " already setup on node "
+ containerManagerAddress);
}
}
Container container_1 = BuilderUtils.NewContainer(containerID, this.nodeId, nodeHttpAddress
, tokenId.GetResource(), null, null);
// DKDC - Doesn't matter
ContainerStatus containerStatus = BuilderUtils.NewContainerStatus(container_1.GetId
(), ContainerState.New, string.Empty, -1000);
applicationContainers.AddItem(container_1);
containerStatusMap[container_1] = containerStatus;
Resources.SubtractFrom(available, tokenId.GetResource());
Resources.AddTo(used, tokenId.GetResource());
if (Log.IsDebugEnabled())
{
Log.Debug("startContainer:" + " node=" + containerManagerAddress + " application="
+ applicationId + " container=" + container_1 + " available=" + available + " used="
+ used);
}
}
StartContainersResponse response = StartContainersResponse.NewInstance(null, null
, null);
return(response);
}
}
// Protected. For tests.
protected internal virtual ContainerManagementProtocol GetContainerMgrProxy(ContainerId
containerId)
{
NodeId node = masterContainer.GetNodeId();
IPEndPoint containerManagerConnectAddress = NetUtils.CreateSocketAddrForHost(node
.GetHost(), node.GetPort());
YarnRPC rpc = GetYarnRPC();
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(containerId
.GetApplicationAttemptId().ToString());
string user = rmContext.GetRMApps()[containerId.GetApplicationAttemptId().GetApplicationId
()].GetUser();
Token token = rmContext.GetNMTokenSecretManager().CreateNMToken(containerId.GetApplicationAttemptId
(), node, user);
currentUser.AddToken(ConverterUtils.ConvertFromYarn(token, containerManagerConnectAddress
));
return(NMProxy.CreateNMProxy <ContainerManagementProtocol>(conf, currentUser, rpc,
containerManagerConnectAddress));
}
/// <summary>Actions to take when the AM container is completed</summary>
/// <param name="containerId">container ID</param>
/// <exception cref="System.IO.IOException">problems</exception>
public virtual void OnAMContainerFinished(ContainerId containerId)
{
Log.Info("AM Container {} finished, purging application attempt records", containerId
);
// remove all application attempt entries
PurgeAppAttemptRecords(containerId.GetApplicationAttemptId());
// also treat as a container finish to remove container
// level records for the AM container
OnContainerFinished(containerId);
}
private static TimelineEntity CreateContainerEntity(ContainerId containerId)
{
TimelineEntity entity = new TimelineEntity();
entity.SetEntityType(ContainerMetricsConstants.EntityType);
entity.SetEntityId(containerId.ToString());
entity.AddPrimaryFilter(ContainerMetricsConstants.ParentPrimariyFilter, containerId
.GetApplicationAttemptId().ToString());
return(entity);
}
/// <exception cref="System.Exception"/>
private StartContainersResponse StartContainer(Context context, ContainerManagerImpl
cm, ContainerId cid, ContainerLaunchContext clc, LogAggregationContext logAggregationContext
)
{
UserGroupInformation user = UserGroupInformation.CreateRemoteUser(cid.GetApplicationAttemptId
().ToString());
StartContainerRequest scReq = StartContainerRequest.NewInstance(clc, TestContainerManager
.CreateContainerToken(cid, 0, context.GetNodeId(), user.GetShortUserName(), context
.GetContainerTokenSecretManager(), logAggregationContext));
IList <StartContainerRequest> scReqList = new AList <StartContainerRequest>();
scReqList.AddItem(scReq);
NMTokenIdentifier nmToken = new NMTokenIdentifier(cid.GetApplicationAttemptId(),
context.GetNodeId(), user.GetShortUserName(), context.GetNMTokenSecretManager().
GetCurrentKey().GetKeyId());
user.AddTokenIdentifier(nmToken);
return(user.DoAs(new _PrivilegedExceptionAction_264(cm, scReqList)));
}
/// <exception cref="System.IO.IOException"/>
public virtual IDictionary <ContainerId, ContainerHistoryData> GetContainers(ApplicationAttemptId
appAttemptId)
{
IDictionary <ContainerId, ContainerHistoryData> historyDataMap = new Dictionary <ContainerId
, ContainerHistoryData>();
FileSystemApplicationHistoryStore.HistoryFileReader hfReader = GetHistoryFileReader
(appAttemptId.GetApplicationId());
try
{
while (hfReader.HasNext())
{
FileSystemApplicationHistoryStore.HistoryFileReader.Entry entry = hfReader.Next();
if (entry.key.id.StartsWith(ConverterUtils.ContainerPrefix))
{
ContainerId containerId = ConverterUtils.ToContainerId(entry.key.id);
if (containerId.GetApplicationAttemptId().Equals(appAttemptId))
{
ContainerHistoryData historyData = historyDataMap[containerId];
if (historyData == null)
{
historyData = ContainerHistoryData.NewInstance(containerId, null, null, null, long.MinValue
, long.MaxValue, null, int.MaxValue, null);
historyDataMap[containerId] = historyData;
}
if (entry.key.suffix.Equals(StartDataSuffix))
{
MergeContainerHistoryData(historyData, ParseContainerStartData(entry.value));
}
else
{
if (entry.key.suffix.Equals(FinishDataSuffix))
{
MergeContainerHistoryData(historyData, ParseContainerFinishData(entry.value));
}
}
}
}
}
Log.Info("Completed reading history information of all conatiners" + " of application attempt "
+ appAttemptId);
}
catch (IOException)
{
Log.Info("Error when reading history information of some containers" + " of application attempt "
+ appAttemptId);
}
finally
{
hfReader.Close();
}
return(historyDataMap);
}
public virtual void TestContainerLogs()
{
WebResource r = Resource();
ContainerId containerId = BuilderUtils.NewContainerId(0, 0, 0, 0);
string containerIdStr = BuilderUtils.NewContainerId(0, 0, 0, 0).ToString();
ApplicationAttemptId appAttemptId = containerId.GetApplicationAttemptId();
ApplicationId appId = appAttemptId.GetApplicationId();
string appIdStr = appId.ToString();
string filename = "logfile1";
string logMessage = "log message\n";
nmContext.GetApplications()[appId] = new ApplicationImpl(null, "user", appId, null
, nmContext);
MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
new Configuration(), "user", appId, 1);
container.SetState(ContainerState.Running);
nmContext.GetContainers()[containerId] = container;
// write out log file
Path path = dirsHandler.GetLogPathForWrite(ContainerLaunch.GetRelativeContainerLogDir
(appIdStr, containerIdStr) + "/" + filename, false);
FilePath logFile = new FilePath(path.ToUri().GetPath());
logFile.DeleteOnExit();
NUnit.Framework.Assert.IsTrue("Failed to create log dir", logFile.GetParentFile()
.Mkdirs());
PrintWriter pw = new PrintWriter(logFile);
pw.Write(logMessage);
pw.Close();
// ask for it
ClientResponse response = r.Path("ws").Path("v1").Path("node").Path("containerlogs"
).Path(containerIdStr).Path(filename).Accept(MediaType.TextPlain).Get <ClientResponse
>();
string responseText = response.GetEntity <string>();
NUnit.Framework.Assert.AreEqual(logMessage, responseText);
// ask for file that doesn't exist
response = r.Path("ws").Path("v1").Path("node").Path("containerlogs").Path(containerIdStr
).Path("uhhh").Accept(MediaType.TextPlain).Get <ClientResponse>();
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.NotFound.GetStatusCode(), response
.GetStatus());
responseText = response.GetEntity <string>();
NUnit.Framework.Assert.IsTrue(responseText.Contains("Cannot find this log on the local disk."
));
// After container is completed, it is removed from nmContext
Sharpen.Collections.Remove(nmContext.GetContainers(), containerId);
NUnit.Framework.Assert.IsNull(nmContext.GetContainers()[containerId]);
response = r.Path("ws").Path("v1").Path("node").Path("containerlogs").Path(containerIdStr
).Path(filename).Accept(MediaType.TextPlain).Get <ClientResponse>();
responseText = response.GetEntity <string>();
NUnit.Framework.Assert.AreEqual(logMessage, responseText);
}
protected internal virtual void ValidateIds(ApplicationId appId, ApplicationAttemptId
appAttemptId, ContainerId containerId)
{
if (!appAttemptId.GetApplicationId().Equals(appId))
{
throw new NotFoundException("appId and appAttemptId don't match");
}
if (containerId != null && !containerId.GetApplicationAttemptId().Equals(appAttemptId
))
{
throw new NotFoundException("appAttemptId and containerId don't match");
}
}
/// <exception cref="System.IO.IOException"/>
private static ContainerTokenIdentifier CreateContainerTokenId(ContainerId cid, NodeId
nodeId, string user, NMContainerTokenSecretManager secretMgr)
{
long rmid = cid.GetApplicationAttemptId().GetApplicationId().GetClusterTimestamp(
);
ContainerTokenIdentifier ctid = new ContainerTokenIdentifier(cid, nodeId.ToString
(), user, BuilderUtils.NewResource(1024, 1), Runtime.CurrentTimeMillis() + 100000L
, secretMgr.GetCurrentKey().GetKeyId(), rmid, Priority.NewInstance(0), 0);
Org.Apache.Hadoop.Yarn.Api.Records.Token token = BuilderUtils.NewContainerToken(nodeId
, secretMgr.CreatePassword(ctid), ctid);
return(BuilderUtils.NewContainerTokenIdentifier(token));
}
GetCMProxy(string containerMgrBindAddr, ContainerId containerId)
{
IPEndPoint addr = NetUtils.GetConnectAddress(this._enclosing._enclosing.server);
string containerManagerBindAddr = addr.GetHostName() + ":" + addr.Port;
Token token = this._enclosing.tokenSecretManager.CreateNMToken(containerId.GetApplicationAttemptId
(), NodeId.NewInstance(addr.GetHostName(), addr.Port), "user");
ContainerManagementProtocolProxy cmProxy = new ContainerManagementProtocolProxy(this
._enclosing._enclosing.conf);
ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData proxy = new
ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData(this, YarnRPC
.Create(this._enclosing._enclosing.conf), containerManagerBindAddr, containerId,
token);
return proxy;
}
GetApplicationForContainer(ContainerId containerId, Context context)
{
ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId
();
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
application = context.GetApplications()[applicationId];
if (application == null)
{
throw new NotFoundException("Unknown container. Container either has not started or "
+ "has already completed or " + "doesn't belong to this node at all.");
}
return(application);
}
public virtual ContainerHistoryData GetContainer(ContainerId containerId)
{
IDictionary <ContainerId, ContainerHistoryData> subMap = containerData[containerId
.GetApplicationAttemptId()];
if (subMap == null)
{
return(null);
}
else
{
return(subMap[containerId]);
}
}
private void StopContainer(ContainerId containerId, int exitCode)
{
// A container is complete. Put this containers' logs up for aggregation if
// this containers' logs are needed.
AppLogAggregator aggregator = this.appLogAggregators[containerId.GetApplicationAttemptId
().GetApplicationId()];
if (aggregator == null)
{
Log.Warn("Log aggregation is not initialized for " + containerId + ", did it fail to start?"
);
return;
}
aggregator.StartContainerLogAggregation(containerId, exitCode == 0);
}
protected internal virtual IList <ContainerStatus> GetContainerStatuses()
{
IList <ContainerStatus> containerStatuses = new AList <ContainerStatus>();
foreach (Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container in this.context.GetContainers().Values)
{
ContainerId containerId = container.GetContainerId();
ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId
();
ContainerStatus containerStatus = container.CloneAndGetContainerStatus();
if (containerStatus.GetState() == ContainerState.Complete)
{
if (IsApplicationStopped(applicationId))
{
if (Log.IsDebugEnabled())
{
Log.Debug(applicationId + " is completing, " + " remove " + containerId + " from NM context."
);
}
Sharpen.Collections.Remove(context.GetContainers(), containerId);
pendingCompletedContainers[containerId] = containerStatus;
}
else
{
if (!IsContainerRecentlyStopped(containerId))
{
pendingCompletedContainers[containerId] = containerStatus;
}
}
// Adding to finished containers cache. Cache will keep it around at
// least for #durationToTrackStoppedContainers duration. In the
// subsequent call to stop container it will get removed from cache.
AddCompletedContainer(containerId);
}
else
{
containerStatuses.AddItem(containerStatus);
}
}
Sharpen.Collections.AddAll(containerStatuses, pendingCompletedContainers.Values);
if (Log.IsDebugEnabled())
{
Log.Debug("Sending out " + containerStatuses.Count + " container statuses: " + containerStatuses
);
}
return(containerStatuses);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
internal static IList <FilePath> GetContainerLogDirs(ContainerId containerId, LocalDirsHandlerService
dirsHandler)
{
IList <string> logDirs = dirsHandler.GetLogDirsForRead();
IList <FilePath> containerLogDirs = new AList <FilePath>(logDirs.Count);
foreach (string logDir in logDirs)
{
logDir = new FilePath(logDir).ToURI().GetPath();
string appIdStr = ConverterUtils.ToString(containerId.GetApplicationAttemptId().GetApplicationId
());
FilePath appLogDir = new FilePath(logDir, appIdStr);
containerLogDirs.AddItem(new FilePath(appLogDir, containerId.ToString()));
}
return(containerLogDirs);
}
public virtual void TestLogFileWithDriveLetter()
{
ContainerImpl container = Org.Mockito.Mockito.Mock <ContainerImpl>();
ApplicationIdPBImpl appId = Org.Mockito.Mockito.Mock <ApplicationIdPBImpl>();
Org.Mockito.Mockito.When(appId.ToString()).ThenReturn("appId");
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
ApplicationAttemptIdPBImpl appAttemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptIdPBImpl
>();
Org.Mockito.Mockito.When(appAttemptId.GetApplicationId()).ThenReturn(appId);
ConcurrentMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
> applications = new ConcurrentHashMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
applications[appId] = app;
ContainerId containerId = Org.Mockito.Mockito.Mock <ContainerIdPBImpl>();
Org.Mockito.Mockito.When(containerId.ToString()).ThenReturn("containerId");
Org.Mockito.Mockito.When(containerId.GetApplicationAttemptId()).ThenReturn(appAttemptId
);
ConcurrentMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
> containers = new ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
containers[containerId] = container;
LocalDirsHandlerService localDirs = Org.Mockito.Mockito.Mock <LocalDirsHandlerService
>();
Org.Mockito.Mockito.When(localDirs.GetLogPathToRead("appId" + Path.Separator + "containerId"
+ Path.Separator + "fileName")).ThenReturn(new Path("F:/nmlogs/appId/containerId/fileName"
));
NodeManager.NMContext context = Org.Mockito.Mockito.Mock <NodeManager.NMContext>();
Org.Mockito.Mockito.When(context.GetLocalDirsHandler()).ThenReturn(localDirs);
Org.Mockito.Mockito.When(context.GetApplications()).ThenReturn(applications);
Org.Mockito.Mockito.When(context.GetContainers()).ThenReturn(containers);
FilePath logFile = ContainerLogsUtils.GetContainerLogFile(containerId, "fileName"
, null, context);
NUnit.Framework.Assert.IsTrue("logFile lost drive letter " + logFile, logFile.ToString
().IndexOf("F:" + FilePath.separator + "nmlogs") > -1);
}
internal static ContainerId GetMockContainer(long id)
{
ApplicationId appId = Org.Mockito.Mockito.Mock <ApplicationId>();
Org.Mockito.Mockito.When(appId.GetClusterTimestamp()).ThenReturn(314159265L);
Org.Mockito.Mockito.When(appId.GetId()).ThenReturn(3);
ApplicationAttemptId appAttemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptId
>();
Org.Mockito.Mockito.When(appAttemptId.GetApplicationId()).ThenReturn(appId);
Org.Mockito.Mockito.When(appAttemptId.GetAttemptId()).ThenReturn(0);
ContainerId container = Org.Mockito.Mockito.Mock <ContainerId>();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(id);
Org.Mockito.Mockito.When(container.GetApplicationAttemptId()).ThenReturn(appAttemptId
);
return(container);
}
protected internal virtual void ContainerLaunchedOnNode(ContainerId containerId,
SchedulerNode node)
{
lock (this)
{
// Get the application for the finished container
SchedulerApplicationAttempt application = GetCurrentAttemptForContainer(containerId
);
if (application == null)
{
Log.Info("Unknown application " + containerId.GetApplicationAttemptId().GetApplicationId
() + " launched container " + containerId + " on node: " + node);
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeCleanContainerEvent
(node.GetNodeID(), containerId));
return;
}
application.ContainerLaunchedOnNode(containerId, node.GetNodeID());
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual ContainerReport GetContainer(ContainerId containerId)
{
ApplicationHistoryManagerOnTimelineStore.ApplicationReportExt app = GetApplication
(containerId.GetApplicationAttemptId().GetApplicationId(), ApplicationHistoryManagerOnTimelineStore.ApplicationReportField
.UserAndAcls);
CheckAccess(app);
TimelineEntity entity = timelineDataManager.GetEntity(ContainerMetricsConstants.EntityType
, containerId.ToString(), EnumSet.AllOf <TimelineReader.Field>(), UserGroupInformation
.GetLoginUser());
if (entity == null)
{
throw new ContainerNotFoundException("The entity for container " + containerId +
" doesn't exist in the timeline store");
}
else
{
return(ConvertToContainerReport(entity, serverHttpAddress, app.appReport.GetUser(
)));
}
}
protected internal virtual ContainerManagementProtocol NewProxy(YarnRPC rpc, string
containerManagerBindAddr, ContainerId containerId, Token token)
{
if (token == null)
{
throw new SecretManager.InvalidToken("No NMToken sent for " + containerManagerBindAddr
);
}
IPEndPoint cmAddr = NetUtils.CreateSocketAddr(containerManagerBindAddr);
ContainerManagementProtocolProxy.Log.Info("Opening proxy : " + containerManagerBindAddr
);
// the user in createRemoteUser in this context has to be ContainerID
UserGroupInformation user = UserGroupInformation.CreateRemoteUser(containerId.GetApplicationAttemptId
().ToString());
Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> nmToken = ConverterUtils
.ConvertFromYarn(token, cmAddr);
user.AddToken(nmToken);
return(NMProxy.CreateNMProxy <ContainerManagementProtocol>(this._enclosing.conf, user
, rpc, cmAddr));
}
