public virtual ContainerInfo GetContainer(HttpServletRequest req, HttpServletResponse
res, string appId, string appAttemptId, string containerId)
{
UserGroupInformation callerUGI = GetUser(req);
ApplicationId aid = ParseApplicationId(appId);
ApplicationAttemptId aaid = ParseApplicationAttemptId(appAttemptId);
ContainerId cid = ParseContainerId(containerId);
ValidateIds(aid, aaid, cid);
ContainerReport container = null;
try
{
if (callerUGI == null)
{
GetContainerReportRequest request = GetContainerReportRequest.NewInstance(cid);
container = appBaseProt.GetContainerReport(request).GetContainerReport();
}
else
{
container = callerUGI.DoAs(new _PrivilegedExceptionAction_368(this, cid));
}
}
catch (Exception e)
{
RewrapAndThrowException(e);
}
if (container == null)
{
throw new NotFoundException("container with id: " + containerId + " not found");
}
return(new ContainerInfo(container));
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual IDictionary <ContainerId, ContainerReport> GetContainers(ApplicationAttemptId
appAttemptId)
{
ApplicationHistoryManagerOnTimelineStore.ApplicationReportExt app = GetApplication
(appAttemptId.GetApplicationId(), ApplicationHistoryManagerOnTimelineStore.ApplicationReportField
.UserAndAcls);
CheckAccess(app);
TimelineEntities entities = timelineDataManager.GetEntities(ContainerMetricsConstants
.EntityType, new NameValuePair(ContainerMetricsConstants.ParentPrimariyFilter, appAttemptId
.ToString()), null, null, null, null, null, long.MaxValue, EnumSet.AllOf <TimelineReader.Field
>(), UserGroupInformation.GetLoginUser());
IDictionary <ContainerId, ContainerReport> containers = new LinkedHashMap <ContainerId
, ContainerReport>();
if (entities != null && entities.GetEntities() != null)
{
foreach (TimelineEntity entity in entities.GetEntities())
{
ContainerReport container = ConvertToContainerReport(entity, serverHttpAddress, app
.appReport.GetUser());
containers[container.GetContainerId()] = container;
}
}
return(containers);
}
/// <exception cref="System.Exception"/>
public virtual void TestGetContainerReportOnHA()
{
ContainerReport report = client.GetContainerReport(cluster.CreateFakeContainerId(
));
NUnit.Framework.Assert.IsTrue(report != null);
NUnit.Framework.Assert.AreEqual(cluster.CreateFakeContainerReport(), report);
}
public static GetContainerReportResponse NewInstance(ContainerReport containerReport
)
{
GetContainerReportResponse response = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<GetContainerReportResponse>();
response.SetContainerReport(containerReport);
return(response);
}
public override void SetContainerReport(ContainerReport containerReport)
{
MaybeInitBuilder();
if (containerReport == null)
{
builder.ClearContainerReport();
}
this.containerReport = containerReport;
}
private ContainerReport ConvertToContainerReport(ContainerHistoryData containerHistory
, string user)
{
// If the container has the aggregated log, add the server root url
string logUrl = WebAppUtils.GetAggregatedLogURL(serverHttpAddress, containerHistory
.GetAssignedNode().ToString(), containerHistory.GetContainerId().ToString(), containerHistory
.GetContainerId().ToString(), user);
return(ContainerReport.NewInstance(containerHistory.GetContainerId(), containerHistory
.GetAllocatedResource(), containerHistory.GetAssignedNode(), containerHistory.GetPriority
(), containerHistory.GetStartTime(), containerHistory.GetFinishTime(), containerHistory
.GetDiagnosticsInfo(), logUrl, containerHistory.GetContainerExitStatus(), containerHistory
.GetContainerState(), null));
}
public override ContainerReport GetContainerReport()
{
if (this.containerReport != null)
{
return(this.containerReport);
}
YarnServiceProtos.GetContainerReportResponseProtoOrBuilder p = viaProto ? proto :
builder;
if (!p.HasContainerReport())
{
return(null);
}
this.containerReport = ConvertFromProtoFormat(p.GetContainerReport());
return(this.containerReport);
}
public virtual void TestContainerReport()
{
ApplicationId appId = ApplicationId.NewInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 1);
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, 1);
GetContainerReportRequest request = GetContainerReportRequest.NewInstance(containerId
);
GetContainerReportResponse response = clientService.GetContainerReport(request);
ContainerReport container = response.GetContainerReport();
NUnit.Framework.Assert.IsNotNull(container);
NUnit.Framework.Assert.AreEqual(containerId, container.GetContainerId());
NUnit.Framework.Assert.AreEqual("http://0.0.0.0:8188/applicationhistory/logs/" +
"test host:100/container_0_0001_01_000001/" + "container_0_0001_01_000001/user1"
, container.GetLogUrl());
}
public virtual ContainerReport CreateContainerReport()
{
this.readLock.Lock();
ContainerReport containerReport = null;
try
{
containerReport = ContainerReport.NewInstance(this.GetContainerId(), this.GetAllocatedResource
(), this.GetAllocatedNode(), this.GetAllocatedPriority(), this.GetCreationTime()
, this.GetFinishTime(), this.GetDiagnosticsInfo(), this.GetLogURL(), this.GetContainerExitStatus
(), this.GetContainerState(), this.GetNodeHttpAddress());
}
finally
{
this.readLock.Unlock();
}
return(containerReport);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual void TestGetContainerReport()
{
Configuration conf = new Configuration();
AHSClient client = new TestAHSClient.MockAHSClient();
client.Init(conf);
client.Start();
IList <ApplicationReport> expectedReports = ((TestAHSClient.MockAHSClient)client).
GetReports();
ApplicationId applicationId = ApplicationId.NewInstance(1234, 5);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 1);
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, 1);
ContainerReport report = client.GetContainerReport(containerId);
NUnit.Framework.Assert.IsNotNull(report);
NUnit.Framework.Assert.AreEqual(report.GetContainerId().ToString(), (ContainerId.
NewContainerId(expectedReports[0].GetCurrentApplicationAttemptId(), 1)).ToString
());
client.Stop();
}
public ContainerInfo(ContainerReport container)
{
// JAXB needs this
containerId = container.GetContainerId().ToString();
if (container.GetAllocatedResource() != null)
{
allocatedMB = container.GetAllocatedResource().GetMemory();
allocatedVCores = container.GetAllocatedResource().GetVirtualCores();
}
if (container.GetAssignedNode() != null)
{
assignedNodeId = container.GetAssignedNode().ToString();
}
priority = container.GetPriority().GetPriority();
startedTime = container.GetCreationTime();
finishedTime = container.GetFinishTime();
elapsedTime = Times.Elapsed(startedTime, finishedTime);
diagnosticsInfo = container.GetDiagnosticsInfo();
logUrl = container.GetLogUrl();
containerExitStatus = container.GetContainerExitStatus();
containerState = container.GetContainerState();
nodeHttpAddress = container.GetNodeHttpAddress();
}
private YarnProtos.ContainerReportProto ConvertToProtoFormat(ContainerReport t)
{
return(((ContainerReportPBImpl)t).GetProto());
}
public abstract void SetContainerReport(ContainerReport containerReport);
/// <summary>Prints the container report for an container id.</summary>
/// <param name="containerId"/>
/// <returns>exitCode</returns>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private int PrintContainerReport(string containerId)
{
ContainerReport containerReport = null;
try
{
containerReport = client.GetContainerReport((ConverterUtils.ToContainerId(containerId
)));
}
catch (ApplicationNotFoundException)
{
sysout.WriteLine("Application for Container with id '" + containerId + "' doesn't exist in RM or Timeline Server."
);
return(-1);
}
catch (ApplicationAttemptNotFoundException)
{
sysout.WriteLine("Application Attempt for Container with id '" + containerId + "' doesn't exist in RM or Timeline Server."
);
return(-1);
}
catch (ContainerNotFoundException)
{
sysout.WriteLine("Container with id '" + containerId + "' doesn't exist in RM or Timeline Server."
);
return(-1);
}
// Use PrintWriter.println, which uses correct platform line ending.
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter containerReportStr = new PrintWriter(new OutputStreamWriter(baos, Sharpen.Extensions.GetEncoding
("UTF-8")));
if (containerReport != null)
{
containerReportStr.WriteLine("Container Report : ");
containerReportStr.Write("\tContainer-Id : ");
containerReportStr.WriteLine(containerReport.GetContainerId());
containerReportStr.Write("\tStart-Time : ");
containerReportStr.WriteLine(containerReport.GetCreationTime());
containerReportStr.Write("\tFinish-Time : ");
containerReportStr.WriteLine(containerReport.GetFinishTime());
containerReportStr.Write("\tState : ");
containerReportStr.WriteLine(containerReport.GetContainerState());
containerReportStr.Write("\tLOG-URL : ");
containerReportStr.WriteLine(containerReport.GetLogUrl());
containerReportStr.Write("\tHost : ");
containerReportStr.WriteLine(containerReport.GetAssignedNode());
containerReportStr.Write("\tNodeHttpAddress : ");
containerReportStr.WriteLine(containerReport.GetNodeHttpAddress() == null ? "N/A"
: containerReport.GetNodeHttpAddress());
containerReportStr.Write("\tDiagnostics : ");
containerReportStr.Write(containerReport.GetDiagnosticsInfo());
}
else
{
containerReportStr.Write("Container with id '" + containerId + "' doesn't exist in Timeline Server."
);
containerReportStr.Close();
sysout.WriteLine(baos.ToString("UTF-8"));
return(-1);
}
containerReportStr.Close();
sysout.WriteLine(baos.ToString("UTF-8"));
return(0);
}
public virtual ContainerReport CreateFakeContainerReport()
{
return(ContainerReport.NewInstance(this.CreateFakeContainerId(), null, NodeId.NewInstance
("localhost", 0), null, 1000l, 1200l, string.Empty, string.Empty, 0, ContainerState
.Complete, "http://" + NodeId.NewInstance("localhost", 0).ToString()));
}
protected override void Render(HtmlBlock.Block html)
{
string containerid = $(YarnWebParams.ContainerId);
if (containerid.IsEmpty())
{
Puts("Bad request: requires container ID");
return;
}
ContainerId containerId = null;
try
{
containerId = ConverterUtils.ToContainerId(containerid);
}
catch (ArgumentException)
{
Puts("Invalid container ID: " + containerid);
return;
}
UserGroupInformation callerUGI = GetCallerUGI();
ContainerReport containerReport = null;
try
{
GetContainerReportRequest request = GetContainerReportRequest.NewInstance(containerId
);
if (callerUGI == null)
{
containerReport = appBaseProt.GetContainerReport(request).GetContainerReport();
}
else
{
containerReport = callerUGI.DoAs(new _PrivilegedExceptionAction_78(this, request)
);
}
}
catch (Exception e)
{
string message = "Failed to read the container " + containerid + ".";
Log.Error(message, e);
html.P().(message).();
return;
}
if (containerReport == null)
{
Puts("Container not found: " + containerid);
return;
}
ContainerInfo container = new ContainerInfo(containerReport);
SetTitle(StringHelper.Join("Container ", containerid));
Info("Container Overview").("Container State:", container.GetContainerState() ==
null ? Unavailable : container.GetContainerState()).("Exit Status:", container.GetContainerExitStatus
()).("Node:", container.GetNodeHttpAddress() == null ? "#" : container.GetNodeHttpAddress
(), container.GetNodeHttpAddress() == null ? "N/A" : container.GetNodeHttpAddress
()).("Priority:", container.GetPriority()).("Started:", Times.Format(container.GetStartedTime
())).("Elapsed:", StringUtils.FormatTime(Times.Elapsed(container.GetStartedTime(
), container.GetFinishedTime()))).("Resource:", container.GetAllocatedMB() + " Memory, "
+ container.GetAllocatedVCores() + " VCores").("Logs:", container.GetLogUrl() ==
null ? "#" : container.GetLogUrl(), container.GetLogUrl() == null ? "N/A" : "Logs"
).("Diagnostics:", container.GetDiagnosticsInfo() == null ? string.Empty : container
.GetDiagnosticsInfo());
html.(typeof(InfoBlock));
}
private void CreateAppReports()
{
ApplicationId applicationId = ApplicationId.NewInstance(1234, 5);
ApplicationReport newApplicationReport = ApplicationReport.NewInstance(applicationId
, ApplicationAttemptId.NewInstance(applicationId, 1), "user", "queue", "appname"
, "host", 124, null, YarnApplicationState.Running, "diagnostics", "url", 0, 0, FinalApplicationStatus
.Succeeded, null, "N/A", 0.53789f, "YARN", null);
IList <ApplicationReport> applicationReports = new AList <ApplicationReport>();
applicationReports.AddItem(newApplicationReport);
IList <ApplicationAttemptReport> appAttempts = new AList <ApplicationAttemptReport>
();
ApplicationAttemptReport attempt = ApplicationAttemptReport.NewInstance(ApplicationAttemptId
.NewInstance(applicationId, 1), "host", 124, "url", "oUrl", "diagnostics", YarnApplicationAttemptState
.Finished, ContainerId.NewContainerId(newApplicationReport.GetCurrentApplicationAttemptId
(), 1));
appAttempts.AddItem(attempt);
ApplicationAttemptReport attempt1 = ApplicationAttemptReport.NewInstance(ApplicationAttemptId
.NewInstance(applicationId, 2), "host", 124, "url", "oUrl", "diagnostics", YarnApplicationAttemptState
.Finished, ContainerId.NewContainerId(newApplicationReport.GetCurrentApplicationAttemptId
(), 2));
appAttempts.AddItem(attempt1);
attempts[applicationId] = appAttempts;
IList <ContainerReport> containerReports = new AList <ContainerReport>();
ContainerReport container = ContainerReport.NewInstance(ContainerId.NewContainerId
(attempt.GetApplicationAttemptId(), 1), null, NodeId.NewInstance("host", 1234),
Priority.Undefined, 1234, 5678, "diagnosticInfo", "logURL", 0, ContainerState.Complete
, "http://" + NodeId.NewInstance("host", 2345).ToString());
containerReports.AddItem(container);
ContainerReport container1 = ContainerReport.NewInstance(ContainerId.NewContainerId
(attempt.GetApplicationAttemptId(), 2), null, NodeId.NewInstance("host", 1234),
Priority.Undefined, 1234, 5678, "diagnosticInfo", "logURL", 0, ContainerState.Complete
, "http://" + NodeId.NewInstance("host", 2345).ToString());
containerReports.AddItem(container1);
containers[attempt.GetApplicationAttemptId()] = containerReports;
ApplicationId applicationId2 = ApplicationId.NewInstance(1234, 6);
ApplicationReport newApplicationReport2 = ApplicationReport.NewInstance(applicationId2
, ApplicationAttemptId.NewInstance(applicationId2, 2), "user2", "queue2", "appname2"
, "host2", 125, null, YarnApplicationState.Finished, "diagnostics2", "url2", 2,
2, FinalApplicationStatus.Succeeded, null, "N/A", 0.63789f, "NON-YARN", null);
applicationReports.AddItem(newApplicationReport2);
ApplicationId applicationId3 = ApplicationId.NewInstance(1234, 7);
ApplicationReport newApplicationReport3 = ApplicationReport.NewInstance(applicationId3
, ApplicationAttemptId.NewInstance(applicationId3, 3), "user3", "queue3", "appname3"
, "host3", 126, null, YarnApplicationState.Running, "diagnostics3", "url3", 3, 3
, FinalApplicationStatus.Succeeded, null, "N/A", 0.73789f, "MAPREDUCE", null);
applicationReports.AddItem(newApplicationReport3);
ApplicationId applicationId4 = ApplicationId.NewInstance(1234, 8);
ApplicationReport newApplicationReport4 = ApplicationReport.NewInstance(applicationId4
, ApplicationAttemptId.NewInstance(applicationId4, 4), "user4", "queue4", "appname4"
, "host4", 127, null, YarnApplicationState.Failed, "diagnostics4", "url4", 4, 4,
FinalApplicationStatus.Succeeded, null, "N/A", 0.83789f, "NON-MAPREDUCE", null);
applicationReports.AddItem(newApplicationReport4);
reports = applicationReports;
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public override IList <ContainerReport> GetContainers(ApplicationAttemptId applicationAttemptId
)
{
IList <ContainerReport> containersForAttempt = new AList <ContainerReport>();
bool appNotFoundInRM = false;
try
{
GetContainersRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetContainersRequest
>();
request.SetApplicationAttemptId(applicationAttemptId);
GetContainersResponse response = rmClient.GetContainers(request);
Sharpen.Collections.AddAll(containersForAttempt, response.GetContainerList());
}
catch (YarnException e)
{
if (e.GetType() != typeof(ApplicationNotFoundException) || !historyServiceEnabled)
{
// If Application is not in RM and history service is enabled then we
// need to check with history service else throw exception.
throw;
}
appNotFoundInRM = true;
}
if (historyServiceEnabled)
{
// Check with AHS even if found in RM because to capture info of finished
// containers also
IList <ContainerReport> containersListFromAHS = null;
try
{
containersListFromAHS = historyClient.GetContainers(applicationAttemptId);
}
catch (IOException e)
{
// History service access might be enabled but system metrics publisher
// is disabled hence app not found exception is possible
if (appNotFoundInRM)
{
// app not found in bothM and RM then propagate the exception.
throw;
}
}
if (null != containersListFromAHS && containersListFromAHS.Count > 0)
{
// remove duplicates
ICollection <ContainerId> containerIdsToBeKeptFromAHS = new HashSet <ContainerId>();
IEnumerator <ContainerReport> tmpItr = containersListFromAHS.GetEnumerator();
while (tmpItr.HasNext())
{
containerIdsToBeKeptFromAHS.AddItem(tmpItr.Next().GetContainerId());
}
IEnumerator <ContainerReport> rmContainers = containersForAttempt.GetEnumerator();
while (rmContainers.HasNext())
{
ContainerReport tmp = rmContainers.Next();
containerIdsToBeKeptFromAHS.Remove(tmp.GetContainerId());
}
// Remove containers from AHS as container from RM will have latest
// information
if (containerIdsToBeKeptFromAHS.Count > 0 && containersListFromAHS.Count != containerIdsToBeKeptFromAHS
.Count)
{
IEnumerator <ContainerReport> containersFromHS = containersListFromAHS.GetEnumerator
();
while (containersFromHS.HasNext())
{
ContainerReport containerReport = containersFromHS.Next();
if (containerIdsToBeKeptFromAHS.Contains(containerReport.GetContainerId()))
{
containersForAttempt.AddItem(containerReport);
}
}
}
else
{
if (containersListFromAHS.Count == containerIdsToBeKeptFromAHS.Count)
{
Sharpen.Collections.AddAll(containersForAttempt, containersListFromAHS);
}
}
}
}
return(containersForAttempt);
}
private static ContainerReport ConvertToContainerReport(TimelineEntity entity, string
serverHttpAddress, string user)
{
int allocatedMem = 0;
int allocatedVcore = 0;
string allocatedHost = null;
int allocatedPort = -1;
int allocatedPriority = 0;
long createdTime = 0;
long finishedTime = 0;
string diagnosticsInfo = null;
int exitStatus = ContainerExitStatus.Invalid;
ContainerState state = null;
string nodeHttpAddress = null;
IDictionary <string, object> entityInfo = entity.GetOtherInfo();
if (entityInfo != null)
{
if (entityInfo.Contains(ContainerMetricsConstants.AllocatedMemoryEntityInfo))
{
allocatedMem = (int)entityInfo[ContainerMetricsConstants.AllocatedMemoryEntityInfo
];
}
if (entityInfo.Contains(ContainerMetricsConstants.AllocatedVcoreEntityInfo))
{
allocatedVcore = (int)entityInfo[ContainerMetricsConstants.AllocatedVcoreEntityInfo
];
}
if (entityInfo.Contains(ContainerMetricsConstants.AllocatedHostEntityInfo))
{
allocatedHost = entityInfo[ContainerMetricsConstants.AllocatedHostEntityInfo].ToString
();
}
if (entityInfo.Contains(ContainerMetricsConstants.AllocatedPortEntityInfo))
{
allocatedPort = (int)entityInfo[ContainerMetricsConstants.AllocatedPortEntityInfo
];
}
if (entityInfo.Contains(ContainerMetricsConstants.AllocatedPriorityEntityInfo))
{
allocatedPriority = (int)entityInfo[ContainerMetricsConstants.AllocatedPriorityEntityInfo
];
}
if (entityInfo.Contains(ContainerMetricsConstants.AllocatedHostHttpAddressEntityInfo
))
{
nodeHttpAddress = (string)entityInfo[ContainerMetricsConstants.AllocatedHostHttpAddressEntityInfo
];
}
}
IList <TimelineEvent> events = entity.GetEvents();
if (events != null)
{
foreach (TimelineEvent @event in events)
{
if (@event.GetEventType().Equals(ContainerMetricsConstants.CreatedEventType))
{
createdTime = @event.GetTimestamp();
}
else
{
if (@event.GetEventType().Equals(ContainerMetricsConstants.FinishedEventType))
{
finishedTime = @event.GetTimestamp();
IDictionary <string, object> eventInfo = @event.GetEventInfo();
if (eventInfo == null)
{
continue;
}
if (eventInfo.Contains(ContainerMetricsConstants.DiagnosticsInfoEventInfo))
{
diagnosticsInfo = eventInfo[ContainerMetricsConstants.DiagnosticsInfoEventInfo].ToString
();
}
if (eventInfo.Contains(ContainerMetricsConstants.ExitStatusEventInfo))
{
exitStatus = (int)eventInfo[ContainerMetricsConstants.ExitStatusEventInfo];
}
if (eventInfo.Contains(ContainerMetricsConstants.StateEventInfo))
{
state = ContainerState.ValueOf(eventInfo[ContainerMetricsConstants.StateEventInfo
].ToString());
}
}
}
}
}
NodeId allocatedNode = NodeId.NewInstance(allocatedHost, allocatedPort);
ContainerId containerId = ConverterUtils.ToContainerId(entity.GetEntityId());
string logUrl = WebAppUtils.GetAggregatedLogURL(serverHttpAddress, allocatedNode.
ToString(), containerId.ToString(), containerId.ToString(), user);
return(ContainerReport.NewInstance(ConverterUtils.ToContainerId(entity.GetEntityId
()), Resource.NewInstance(allocatedMem, allocatedVcore), NodeId.NewInstance(allocatedHost
, allocatedPort), Priority.NewInstance(allocatedPriority), createdTime, finishedTime
, diagnosticsInfo, logUrl, exitStatus, state, nodeHttpAddress));
}