/// <exception cref="System.IO.IOException"/>
private static void ReadContainerLogs(DataInputStream valueStream, TextWriter @out
, long logUploadedTime)
{
byte[] buf = new byte[65535];
string fileType = valueStream.ReadUTF();
string fileLengthStr = valueStream.ReadUTF();
long fileLength = long.Parse(fileLengthStr);
@out.Write("LogType:");
@out.WriteLine(fileType);
if (logUploadedTime != -1)
{
@out.Write("Log Upload Time:");
@out.WriteLine(Times.Format(logUploadedTime));
}
@out.Write("LogLength:");
@out.WriteLine(fileLengthStr);
@out.WriteLine("Log Contents:");
long curRead = 0;
long pendingRead = fileLength - curRead;
int toRead = pendingRead > buf.Length ? buf.Length : (int)pendingRead;
int len = valueStream.Read(buf, 0, toRead);
while (len != -1 && curRead < fileLength)
{
@out.Write(buf, 0, len);
curRead += len;
pendingRead = fileLength - curRead;
toRead = pendingRead > buf.Length ? buf.Length : (int)pendingRead;
len = valueStream.Read(buf, 0, toRead);
}
@out.WriteLine("End of LogType:" + fileType);
@out.WriteLine(string.Empty);
}
protected override void Render(HtmlBlock.Block html)
{
SCMMetricsInfo metricsInfo = new SCMMetricsInfo(CleanerMetrics.GetInstance(), ClientSCMMetrics
.GetInstance(), SharedCacheUploaderMetrics.GetInstance());
Info("Shared Cache Manager overview").("Started on:", Times.Format(scm.GetStartTime
())).("Cache hits: ", metricsInfo.GetCacheHits()).("Cache misses: ", metricsInfo
.GetCacheMisses()).("Cache releases: ", metricsInfo.GetCacheReleases()).("Accepted uploads: "
, metricsInfo.GetAcceptedUploads()).("Rejected uploads: ", metricsInfo.GetRejectUploads
()).("Deleted files by the cleaner: ", metricsInfo.GetTotalDeletedFiles()).("Processed files by the cleaner: "
, metricsInfo.GetTotalProcessedFiles());
html.(typeof(InfoBlock));
}
/// <summary>Lists the containers matching the given application attempts</summary>
/// <param name="appAttemptId"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private void ListContainers(string appAttemptId)
{
PrintWriter writer = new PrintWriter(new OutputStreamWriter(sysout, Sharpen.Extensions.GetEncoding
("UTF-8")));
IList <ContainerReport> appsReport = client.GetContainers(ConverterUtils.ToApplicationAttemptId
(appAttemptId));
writer.WriteLine("Total number of containers " + ":" + appsReport.Count);
writer.Printf(ContainerPattern, "Container-Id", "Start Time", "Finish Time", "State"
, "Host", "Node Http Address", "LOG-URL");
foreach (ContainerReport containerReport in appsReport)
{
writer.Printf(ContainerPattern, containerReport.GetContainerId(), Times.Format(containerReport
.GetCreationTime()), Times.Format(containerReport.GetFinishTime()), containerReport
.GetContainerState(), containerReport.GetAssignedNode(), containerReport.GetNodeHttpAddress
() == null ? "N/A" : containerReport.GetNodeHttpAddress(), containerReport.GetLogUrl
());
}
writer.Flush();
}
/// <exception cref="System.IO.IOException"/>
private bool ReadContainerLogs(HtmlBlock.Block html, AggregatedLogFormat.ContainerLogsReader
logReader, AggregatedLogsBlock.LogLimits logLimits, string desiredLogType, long
logUpLoadTime)
{
int bufferSize = 65536;
char[] cbuf = new char[bufferSize];
bool foundLog = false;
string logType = logReader.NextLog();
while (logType != null)
{
if (desiredLogType == null || desiredLogType.IsEmpty() || desiredLogType.Equals(logType
))
{
long logLength = logReader.GetCurrentLogLength();
if (foundLog)
{
html.Pre().("\n\n").();
}
html.P().("Log Type: " + logType).();
html.P().("Log Upload Time: " + Times.Format(logUpLoadTime)).();
html.P().("Log Length: " + System.Convert.ToString(logLength)).();
long start = logLimits.start < 0 ? logLength + logLimits.start : logLimits.start;
start = start < 0 ? 0 : start;
start = start > logLength ? logLength : start;
long end = logLimits.end < 0 ? logLength + logLimits.end : logLimits.end;
end = end < 0 ? 0 : end;
end = end > logLength ? logLength : end;
end = end < start ? start : end;
long toRead = end - start;
if (toRead < logLength)
{
html.P().("Showing " + toRead + " bytes of " + logLength + " total. Click ").A(Url
("logs", $(YarnWebParams.NmNodename), $(YarnWebParams.ContainerId), $(YarnWebParams
.EntityString), $(YarnWebParams.AppOwner), logType, "?start=0"), "here").(" for the full log."
).();
}
long totalSkipped = 0;
while (totalSkipped < start)
{
long ret = logReader.Skip(start - totalSkipped);
if (ret == 0)
{
//Read one byte
int nextByte = logReader.Read();
// Check if we have reached EOF
if (nextByte == -1)
{
throw new IOException("Premature EOF from container log");
}
ret = 1;
}
totalSkipped += ret;
}
int len = 0;
int currentToRead = toRead > bufferSize ? bufferSize : (int)toRead;
Hamlet.PRE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> pre = html.Pre();
while (toRead > 0 && (len = logReader.Read(cbuf, 0, currentToRead)) > 0)
{
pre.(new string(cbuf, 0, len));
toRead = toRead - len;
currentToRead = toRead > bufferSize ? bufferSize : (int)toRead;
}
pre.();
foundLog = true;
}
logType = logReader.NextLog();
}
return(foundLog);
}
/// <summary>Render the /info page with an overview of current application.</summary>
public virtual void Info()
{
AppInfo info = new AppInfo(app, app.context);
Info("Application Master Overview").("Application ID:", info.GetId()).("Application Name:"
, info.GetName()).("User:"******"Started on:", Times.Format(info.GetStartTime
())).("Elasped: ", StringUtils.FormatTime(info.GetElapsedTime()));
Render(typeof(InfoPage));
}
protected override void Render(HtmlBlock.Block html)
{
html.(typeof(MetricsOverviewTable));
ResourceManager rm = GetInstance <ResourceManager>();
ClusterInfo cinfo = new ClusterInfo(rm);
Info("Cluster overview").("Cluster ID:", cinfo.GetClusterId()).("ResourceManager state:"
, cinfo.GetState()).("ResourceManager HA state:", cinfo.GetHAState()).("ResourceManager HA zookeeper connection state:"
, cinfo.GetHAZookeeperConnectionState()).("ResourceManager RMStateStore:", cinfo
.GetRMStateStore()).("ResourceManager started on:", Times.Format(cinfo.GetStartedOn
())).("ResourceManager version:", cinfo.GetRMBuildVersion() + " on " + cinfo.GetRMVersionBuiltOn
()).("Hadoop version:", cinfo.GetHadoopBuildVersion() + " on " + cinfo.GetHadoopVersionBuiltOn
());
html.(typeof(InfoBlock));
}
protected override void Render(HtmlBlock.Block html)
{
string webUiType = $(YarnWebParams.WebUiType);
string aid = $(YarnWebParams.ApplicationId);
if (aid.IsEmpty())
{
Puts("Bad request: requires Application ID");
return;
}
try
{
appID = Apps.ToAppID(aid);
}
catch (Exception)
{
Puts("Invalid Application ID: " + aid);
return;
}
UserGroupInformation callerUGI = GetCallerUGI();
ApplicationReport appReport;
try
{
GetApplicationReportRequest request = GetApplicationReportRequest.NewInstance(appID
);
if (callerUGI == null)
{
appReport = appBaseProt.GetApplicationReport(request).GetApplicationReport();
}
else
{
appReport = callerUGI.DoAs(new _PrivilegedExceptionAction_99(this, request));
}
}
catch (Exception e)
{
string message = "Failed to read the application " + appID + ".";
Log.Error(message, e);
html.P().(message).();
return;
}
if (appReport == null)
{
Puts("Application not found: " + aid);
return;
}
AppInfo app = new AppInfo(appReport);
SetTitle(StringHelper.Join("Application ", aid));
if (webUiType != null && webUiType.Equals(YarnWebParams.RmWebUi) && conf.GetBoolean
(YarnConfiguration.RmWebappUiActionsEnabled, YarnConfiguration.DefaultRmWebappUiActionsEnabled
))
{
// Application Kill
html.Div().Button().$onclick("confirmAction()").B("Kill Application").().();
StringBuilder script = new StringBuilder();
script.Append("function confirmAction() {").Append(" b = confirm(\"Are you sure?\");"
).Append(" if (b == true) {").Append(" $.ajax({").Append(" type: 'PUT',").Append
(" url: '/ws/v1/cluster/apps/").Append(aid).Append("/state',").Append(" contentType: 'application/json',"
).Append(" data: '{\"state\":\"KILLED\"}',").Append(" dataType: 'json'").Append(
" }).done(function(data){").Append(" setTimeout(function(){").Append(" location.href = '/cluster/app/"
).Append(aid).Append("';").Append(" }, 1000);").Append(" }).fail(function(data){"
).Append(" console.log(data);").Append(" });").Append(" }").Append("}");
html.Script().$type("text/javascript").(script.ToString()).();
}
Info("Application Overview").("User:"******"Name:", app.GetName()).("Application Type:"
, app.GetType()).("Application Tags:", app.GetApplicationTags() == null ? string.Empty
: app.GetApplicationTags()).("YarnApplicationState:", app.GetAppState() == null
? Unavailable : ClarifyAppState(app.GetAppState())).("FinalStatus Reported by AM:"
, ClairfyAppFinalStatus(app.GetFinalAppStatus())).("Started:", Times.Format(app.
GetStartedTime())).("Elapsed:", StringUtils.FormatTime(Times.Elapsed(app.GetStartedTime
(), app.GetFinishedTime()))).("Tracking URL:", app.GetTrackingUrl() == null || app
.GetTrackingUrl().Equals(Unavailable) ? null : Root_url(app.GetTrackingUrl()), app
.GetTrackingUrl() == null || app.GetTrackingUrl().Equals(Unavailable) ? "Unassigned"
: app.GetAppState() == YarnApplicationState.Finished || app.GetAppState() == YarnApplicationState
.Failed || app.GetAppState() == YarnApplicationState.Killed ? "History" : "ApplicationMaster"
).("Diagnostics:", app.GetDiagnosticsInfo() == null ? string.Empty : app.GetDiagnosticsInfo
());
ICollection <ApplicationAttemptReport> attempts;
try
{
GetApplicationAttemptsRequest request = GetApplicationAttemptsRequest.NewInstance
(appID);
if (callerUGI == null)
{
attempts = appBaseProt.GetApplicationAttempts(request).GetApplicationAttemptList(
);
}
else
{
attempts = callerUGI.DoAs(new _PrivilegedExceptionAction_196(this, request));
}
}
catch (Exception e)
{
string message = "Failed to read the attempts of the application " + appID + ".";
Log.Error(message, e);
html.P().(message).();
return;
}
CreateApplicationMetricsTable(html);
html.(typeof(InfoBlock));
GenerateApplicationTable(html, callerUGI, attempts);
}
protected override void Render(HtmlBlock.Block html)
{
string containerid = $(YarnWebParams.ContainerId);
if (containerid.IsEmpty())
{
Puts("Bad request: requires container ID");
return;
}
ContainerId containerId = null;
try
{
containerId = ConverterUtils.ToContainerId(containerid);
}
catch (ArgumentException)
{
Puts("Invalid container ID: " + containerid);
return;
}
UserGroupInformation callerUGI = GetCallerUGI();
ContainerReport containerReport = null;
try
{
GetContainerReportRequest request = GetContainerReportRequest.NewInstance(containerId
);
if (callerUGI == null)
{
containerReport = appBaseProt.GetContainerReport(request).GetContainerReport();
}
else
{
containerReport = callerUGI.DoAs(new _PrivilegedExceptionAction_78(this, request)
);
}
}
catch (Exception e)
{
string message = "Failed to read the container " + containerid + ".";
Log.Error(message, e);
html.P().(message).();
return;
}
if (containerReport == null)
{
Puts("Container not found: " + containerid);
return;
}
ContainerInfo container = new ContainerInfo(containerReport);
SetTitle(StringHelper.Join("Container ", containerid));
Info("Container Overview").("Container State:", container.GetContainerState() ==
null ? Unavailable : container.GetContainerState()).("Exit Status:", container.GetContainerExitStatus
()).("Node:", container.GetNodeHttpAddress() == null ? "#" : container.GetNodeHttpAddress
(), container.GetNodeHttpAddress() == null ? "N/A" : container.GetNodeHttpAddress
()).("Priority:", container.GetPriority()).("Started:", Times.Format(container.GetStartedTime
())).("Elapsed:", StringUtils.FormatTime(Times.Elapsed(container.GetStartedTime(
), container.GetFinishedTime()))).("Resource:", container.GetAllocatedMB() + " Memory, "
+ container.GetAllocatedVCores() + " VCores").("Logs:", container.GetLogUrl() ==
null ? "#" : container.GetLogUrl(), container.GetLogUrl() == null ? "N/A" : "Logs"
).("Diagnostics:", container.GetDiagnosticsInfo() == null ? string.Empty : container
.GetDiagnosticsInfo());
html.(typeof(InfoBlock));
}
/// <exception cref="System.Exception"/>
private void TestReadAcontainerLog(bool logUploadedTime)
{
Configuration conf = new Configuration();
FilePath workDir = new FilePath(testWorkDir, "testReadAcontainerLogs1");
Path remoteAppLogFile = new Path(workDir.GetAbsolutePath(), "aggregatedLogFile");
Path srcFileRoot = new Path(workDir.GetAbsolutePath(), "srcFiles");
ContainerId testContainerId = TestContainerId.NewContainerId(1, 1, 1, 1);
Path t = new Path(srcFileRoot, testContainerId.GetApplicationAttemptId().GetApplicationId
().ToString());
Path srcFilePath = new Path(t, testContainerId.ToString());
int numChars = 80000;
// create a sub-folder under srcFilePath
// and create file logs in this sub-folder.
// We only aggregate top level files.
// So, this log file should be ignored.
Path subDir = new Path(srcFilePath, "subDir");
fs.Mkdirs(subDir);
WriteSrcFile(subDir, "logs", numChars);
// create file stderr and stdout in containerLogDir
WriteSrcFile(srcFilePath, "stderr", numChars);
WriteSrcFile(srcFilePath, "stdout", numChars);
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
AggregatedLogFormat.LogWriter logWriter = new AggregatedLogFormat.LogWriter(conf,
remoteAppLogFile, ugi);
AggregatedLogFormat.LogKey logKey = new AggregatedLogFormat.LogKey(testContainerId
);
AggregatedLogFormat.LogValue logValue = new AggregatedLogFormat.LogValue(Collections
.SingletonList(srcFileRoot.ToString()), testContainerId, ugi.GetShortUserName());
// When we try to open FileInputStream for stderr, it will throw out an IOException.
// Skip the log aggregation for stderr.
AggregatedLogFormat.LogValue spyLogValue = Org.Mockito.Mockito.Spy(logValue);
FilePath errorFile = new FilePath((new Path(srcFilePath, "stderr")).ToString());
Org.Mockito.Mockito.DoThrow(new IOException("Mock can not open FileInputStream"))
.When(spyLogValue).SecureOpenFile(errorFile);
logWriter.Append(logKey, spyLogValue);
logWriter.Close();
// make sure permission are correct on the file
FileStatus fsStatus = fs.GetFileStatus(remoteAppLogFile);
NUnit.Framework.Assert.AreEqual("permissions on log aggregation file are wrong",
FsPermission.CreateImmutable((short)0x1a0), fsStatus.GetPermission());
AggregatedLogFormat.LogReader logReader = new AggregatedLogFormat.LogReader(conf,
remoteAppLogFile);
AggregatedLogFormat.LogKey rLogKey = new AggregatedLogFormat.LogKey();
DataInputStream dis = logReader.Next(rLogKey);
TextWriter writer = new StringWriter();
if (logUploadedTime)
{
AggregatedLogFormat.LogReader.ReadAcontainerLogs(dis, writer, Runtime.CurrentTimeMillis
());
}
else
{
AggregatedLogFormat.LogReader.ReadAcontainerLogs(dis, writer);
}
// We should only do the log aggregation for stdout.
// Since we could not open the fileInputStream for stderr, this file is not
// aggregated.
string s = writer.ToString();
int expectedLength = "LogType:stdout".Length + (logUploadedTime ? ("\nLog Upload Time:"
+ Times.Format(Runtime.CurrentTimeMillis())).Length : 0) + ("\nLogLength:" + numChars
).Length + "\nLog Contents:\n".Length + numChars + "\n".Length + "End of LogType:stdout\n"
.Length;
NUnit.Framework.Assert.IsTrue("LogType not matched", s.Contains("LogType:stdout")
);
NUnit.Framework.Assert.IsTrue("log file:stderr should not be aggregated.", !s.Contains
("LogType:stderr"));
NUnit.Framework.Assert.IsTrue("log file:logs should not be aggregated.", !s.Contains
("LogType:logs"));
NUnit.Framework.Assert.IsTrue("LogLength not matched", s.Contains("LogLength:" +
numChars));
NUnit.Framework.Assert.IsTrue("Log Contents not matched", s.Contains("Log Contents"
));
StringBuilder sb = new StringBuilder();
for (int i = 0; i < numChars; i++)
{
sb.Append(filler);
}
string expectedContent = sb.ToString();
NUnit.Framework.Assert.IsTrue("Log content incorrect", s.Contains(expectedContent
));
NUnit.Framework.Assert.AreEqual(expectedLength, s.Length);
}
/// <summary>The content of this page is the attempts block</summary>
/// <returns>AttemptsBlock.class</returns>
protected override Type Content()
{
HistoryInfo info = new HistoryInfo();
Info("History Server").("BuildVersion", info.GetHadoopBuildVersion() + " on " + info
.GetHadoopVersionBuiltOn()).("History Server started on", Times.Format(info.GetStartedOn
()));
return(typeof(InfoBlock));
}
