/// <exception cref="System.IO.IOException"/> private static void ReadContainerLogs(DataInputStream valueStream, TextWriter @out , long logUploadedTime) { byte[] buf = new byte[65535]; string fileType = valueStream.ReadUTF(); string fileLengthStr = valueStream.ReadUTF(); long fileLength = long.Parse(fileLengthStr); @out.Write("LogType:"); @out.WriteLine(fileType); if (logUploadedTime != -1) { @out.Write("Log Upload Time:"); @out.WriteLine(Times.Format(logUploadedTime)); } @out.Write("LogLength:"); @out.WriteLine(fileLengthStr); @out.WriteLine("Log Contents:"); long curRead = 0; long pendingRead = fileLength - curRead; int toRead = pendingRead > buf.Length ? buf.Length : (int)pendingRead; int len = valueStream.Read(buf, 0, toRead); while (len != -1 && curRead < fileLength) { @out.Write(buf, 0, len); curRead += len; pendingRead = fileLength - curRead; toRead = pendingRead > buf.Length ? buf.Length : (int)pendingRead; len = valueStream.Read(buf, 0, toRead); } @out.WriteLine("End of LogType:" + fileType); @out.WriteLine(string.Empty); }
protected override void Render(HtmlBlock.Block html) { SCMMetricsInfo metricsInfo = new SCMMetricsInfo(CleanerMetrics.GetInstance(), ClientSCMMetrics .GetInstance(), SharedCacheUploaderMetrics.GetInstance()); Info("Shared Cache Manager overview").("Started on:", Times.Format(scm.GetStartTime ())).("Cache hits: ", metricsInfo.GetCacheHits()).("Cache misses: ", metricsInfo .GetCacheMisses()).("Cache releases: ", metricsInfo.GetCacheReleases()).("Accepted uploads: " , metricsInfo.GetAcceptedUploads()).("Rejected uploads: ", metricsInfo.GetRejectUploads ()).("Deleted files by the cleaner: ", metricsInfo.GetTotalDeletedFiles()).("Processed files by the cleaner: " , metricsInfo.GetTotalProcessedFiles()); html.(typeof(InfoBlock)); }
/// <summary>Lists the containers matching the given application attempts</summary> /// <param name="appAttemptId"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> private void ListContainers(string appAttemptId) { PrintWriter writer = new PrintWriter(new OutputStreamWriter(sysout, Sharpen.Extensions.GetEncoding ("UTF-8"))); IList <ContainerReport> appsReport = client.GetContainers(ConverterUtils.ToApplicationAttemptId (appAttemptId)); writer.WriteLine("Total number of containers " + ":" + appsReport.Count); writer.Printf(ContainerPattern, "Container-Id", "Start Time", "Finish Time", "State" , "Host", "Node Http Address", "LOG-URL"); foreach (ContainerReport containerReport in appsReport) { writer.Printf(ContainerPattern, containerReport.GetContainerId(), Times.Format(containerReport .GetCreationTime()), Times.Format(containerReport.GetFinishTime()), containerReport .GetContainerState(), containerReport.GetAssignedNode(), containerReport.GetNodeHttpAddress () == null ? "N/A" : containerReport.GetNodeHttpAddress(), containerReport.GetLogUrl ()); } writer.Flush(); }
/// <exception cref="System.IO.IOException"/> private bool ReadContainerLogs(HtmlBlock.Block html, AggregatedLogFormat.ContainerLogsReader logReader, AggregatedLogsBlock.LogLimits logLimits, string desiredLogType, long logUpLoadTime) { int bufferSize = 65536; char[] cbuf = new char[bufferSize]; bool foundLog = false; string logType = logReader.NextLog(); while (logType != null) { if (desiredLogType == null || desiredLogType.IsEmpty() || desiredLogType.Equals(logType )) { long logLength = logReader.GetCurrentLogLength(); if (foundLog) { html.Pre().("\n\n").(); } html.P().("Log Type: " + logType).(); html.P().("Log Upload Time: " + Times.Format(logUpLoadTime)).(); html.P().("Log Length: " + System.Convert.ToString(logLength)).(); long start = logLimits.start < 0 ? logLength + logLimits.start : logLimits.start; start = start < 0 ? 0 : start; start = start > logLength ? logLength : start; long end = logLimits.end < 0 ? logLength + logLimits.end : logLimits.end; end = end < 0 ? 0 : end; end = end > logLength ? logLength : end; end = end < start ? start : end; long toRead = end - start; if (toRead < logLength) { html.P().("Showing " + toRead + " bytes of " + logLength + " total. Click ").A(Url ("logs", $(YarnWebParams.NmNodename), $(YarnWebParams.ContainerId), $(YarnWebParams .EntityString), $(YarnWebParams.AppOwner), logType, "?start=0"), "here").(" for the full log." ).(); } long totalSkipped = 0; while (totalSkipped < start) { long ret = logReader.Skip(start - totalSkipped); if (ret == 0) { //Read one byte int nextByte = logReader.Read(); // Check if we have reached EOF if (nextByte == -1) { throw new IOException("Premature EOF from container log"); } ret = 1; } totalSkipped += ret; } int len = 0; int currentToRead = toRead > bufferSize ? bufferSize : (int)toRead; Hamlet.PRE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> pre = html.Pre(); while (toRead > 0 && (len = logReader.Read(cbuf, 0, currentToRead)) > 0) { pre.(new string(cbuf, 0, len)); toRead = toRead - len; currentToRead = toRead > bufferSize ? bufferSize : (int)toRead; } pre.(); foundLog = true; } logType = logReader.NextLog(); } return(foundLog); }
/// <summary>Render the /info page with an overview of current application.</summary> public virtual void Info() { AppInfo info = new AppInfo(app, app.context); Info("Application Master Overview").("Application ID:", info.GetId()).("Application Name:" , info.GetName()).("User:"******"Started on:", Times.Format(info.GetStartTime ())).("Elasped: ", StringUtils.FormatTime(info.GetElapsedTime())); Render(typeof(InfoPage)); }
protected override void Render(HtmlBlock.Block html) { html.(typeof(MetricsOverviewTable)); ResourceManager rm = GetInstance <ResourceManager>(); ClusterInfo cinfo = new ClusterInfo(rm); Info("Cluster overview").("Cluster ID:", cinfo.GetClusterId()).("ResourceManager state:" , cinfo.GetState()).("ResourceManager HA state:", cinfo.GetHAState()).("ResourceManager HA zookeeper connection state:" , cinfo.GetHAZookeeperConnectionState()).("ResourceManager RMStateStore:", cinfo .GetRMStateStore()).("ResourceManager started on:", Times.Format(cinfo.GetStartedOn ())).("ResourceManager version:", cinfo.GetRMBuildVersion() + " on " + cinfo.GetRMVersionBuiltOn ()).("Hadoop version:", cinfo.GetHadoopBuildVersion() + " on " + cinfo.GetHadoopVersionBuiltOn ()); html.(typeof(InfoBlock)); }
protected override void Render(HtmlBlock.Block html) { string webUiType = $(YarnWebParams.WebUiType); string aid = $(YarnWebParams.ApplicationId); if (aid.IsEmpty()) { Puts("Bad request: requires Application ID"); return; } try { appID = Apps.ToAppID(aid); } catch (Exception) { Puts("Invalid Application ID: " + aid); return; } UserGroupInformation callerUGI = GetCallerUGI(); ApplicationReport appReport; try { GetApplicationReportRequest request = GetApplicationReportRequest.NewInstance(appID ); if (callerUGI == null) { appReport = appBaseProt.GetApplicationReport(request).GetApplicationReport(); } else { appReport = callerUGI.DoAs(new _PrivilegedExceptionAction_99(this, request)); } } catch (Exception e) { string message = "Failed to read the application " + appID + "."; Log.Error(message, e); html.P().(message).(); return; } if (appReport == null) { Puts("Application not found: " + aid); return; } AppInfo app = new AppInfo(appReport); SetTitle(StringHelper.Join("Application ", aid)); if (webUiType != null && webUiType.Equals(YarnWebParams.RmWebUi) && conf.GetBoolean (YarnConfiguration.RmWebappUiActionsEnabled, YarnConfiguration.DefaultRmWebappUiActionsEnabled )) { // Application Kill html.Div().Button().$onclick("confirmAction()").B("Kill Application").().(); StringBuilder script = new StringBuilder(); script.Append("function confirmAction() {").Append(" b = confirm(\"Are you sure?\");" ).Append(" if (b == true) {").Append(" $.ajax({").Append(" type: 'PUT',").Append (" url: '/ws/v1/cluster/apps/").Append(aid).Append("/state',").Append(" contentType: 'application/json'," ).Append(" data: '{\"state\":\"KILLED\"}',").Append(" dataType: 'json'").Append( " }).done(function(data){").Append(" setTimeout(function(){").Append(" location.href = '/cluster/app/" ).Append(aid).Append("';").Append(" }, 1000);").Append(" }).fail(function(data){" ).Append(" console.log(data);").Append(" });").Append(" }").Append("}"); html.Script().$type("text/javascript").(script.ToString()).(); } Info("Application Overview").("User:"******"Name:", app.GetName()).("Application Type:" , app.GetType()).("Application Tags:", app.GetApplicationTags() == null ? string.Empty : app.GetApplicationTags()).("YarnApplicationState:", app.GetAppState() == null ? Unavailable : ClarifyAppState(app.GetAppState())).("FinalStatus Reported by AM:" , ClairfyAppFinalStatus(app.GetFinalAppStatus())).("Started:", Times.Format(app. GetStartedTime())).("Elapsed:", StringUtils.FormatTime(Times.Elapsed(app.GetStartedTime (), app.GetFinishedTime()))).("Tracking URL:", app.GetTrackingUrl() == null || app .GetTrackingUrl().Equals(Unavailable) ? null : Root_url(app.GetTrackingUrl()), app .GetTrackingUrl() == null || app.GetTrackingUrl().Equals(Unavailable) ? "Unassigned" : app.GetAppState() == YarnApplicationState.Finished || app.GetAppState() == YarnApplicationState .Failed || app.GetAppState() == YarnApplicationState.Killed ? "History" : "ApplicationMaster" ).("Diagnostics:", app.GetDiagnosticsInfo() == null ? string.Empty : app.GetDiagnosticsInfo ()); ICollection <ApplicationAttemptReport> attempts; try { GetApplicationAttemptsRequest request = GetApplicationAttemptsRequest.NewInstance (appID); if (callerUGI == null) { attempts = appBaseProt.GetApplicationAttempts(request).GetApplicationAttemptList( ); } else { attempts = callerUGI.DoAs(new _PrivilegedExceptionAction_196(this, request)); } } catch (Exception e) { string message = "Failed to read the attempts of the application " + appID + "."; Log.Error(message, e); html.P().(message).(); return; } CreateApplicationMetricsTable(html); html.(typeof(InfoBlock)); GenerateApplicationTable(html, callerUGI, attempts); }
protected override void Render(HtmlBlock.Block html) { string containerid = $(YarnWebParams.ContainerId); if (containerid.IsEmpty()) { Puts("Bad request: requires container ID"); return; } ContainerId containerId = null; try { containerId = ConverterUtils.ToContainerId(containerid); } catch (ArgumentException) { Puts("Invalid container ID: " + containerid); return; } UserGroupInformation callerUGI = GetCallerUGI(); ContainerReport containerReport = null; try { GetContainerReportRequest request = GetContainerReportRequest.NewInstance(containerId ); if (callerUGI == null) { containerReport = appBaseProt.GetContainerReport(request).GetContainerReport(); } else { containerReport = callerUGI.DoAs(new _PrivilegedExceptionAction_78(this, request) ); } } catch (Exception e) { string message = "Failed to read the container " + containerid + "."; Log.Error(message, e); html.P().(message).(); return; } if (containerReport == null) { Puts("Container not found: " + containerid); return; } ContainerInfo container = new ContainerInfo(containerReport); SetTitle(StringHelper.Join("Container ", containerid)); Info("Container Overview").("Container State:", container.GetContainerState() == null ? Unavailable : container.GetContainerState()).("Exit Status:", container.GetContainerExitStatus ()).("Node:", container.GetNodeHttpAddress() == null ? "#" : container.GetNodeHttpAddress (), container.GetNodeHttpAddress() == null ? "N/A" : container.GetNodeHttpAddress ()).("Priority:", container.GetPriority()).("Started:", Times.Format(container.GetStartedTime ())).("Elapsed:", StringUtils.FormatTime(Times.Elapsed(container.GetStartedTime( ), container.GetFinishedTime()))).("Resource:", container.GetAllocatedMB() + " Memory, " + container.GetAllocatedVCores() + " VCores").("Logs:", container.GetLogUrl() == null ? "#" : container.GetLogUrl(), container.GetLogUrl() == null ? "N/A" : "Logs" ).("Diagnostics:", container.GetDiagnosticsInfo() == null ? string.Empty : container .GetDiagnosticsInfo()); html.(typeof(InfoBlock)); }
/// <exception cref="System.Exception"/> private void TestReadAcontainerLog(bool logUploadedTime) { Configuration conf = new Configuration(); FilePath workDir = new FilePath(testWorkDir, "testReadAcontainerLogs1"); Path remoteAppLogFile = new Path(workDir.GetAbsolutePath(), "aggregatedLogFile"); Path srcFileRoot = new Path(workDir.GetAbsolutePath(), "srcFiles"); ContainerId testContainerId = TestContainerId.NewContainerId(1, 1, 1, 1); Path t = new Path(srcFileRoot, testContainerId.GetApplicationAttemptId().GetApplicationId ().ToString()); Path srcFilePath = new Path(t, testContainerId.ToString()); int numChars = 80000; // create a sub-folder under srcFilePath // and create file logs in this sub-folder. // We only aggregate top level files. // So, this log file should be ignored. Path subDir = new Path(srcFilePath, "subDir"); fs.Mkdirs(subDir); WriteSrcFile(subDir, "logs", numChars); // create file stderr and stdout in containerLogDir WriteSrcFile(srcFilePath, "stderr", numChars); WriteSrcFile(srcFilePath, "stdout", numChars); UserGroupInformation ugi = UserGroupInformation.GetCurrentUser(); AggregatedLogFormat.LogWriter logWriter = new AggregatedLogFormat.LogWriter(conf, remoteAppLogFile, ugi); AggregatedLogFormat.LogKey logKey = new AggregatedLogFormat.LogKey(testContainerId ); AggregatedLogFormat.LogValue logValue = new AggregatedLogFormat.LogValue(Collections .SingletonList(srcFileRoot.ToString()), testContainerId, ugi.GetShortUserName()); // When we try to open FileInputStream for stderr, it will throw out an IOException. // Skip the log aggregation for stderr. AggregatedLogFormat.LogValue spyLogValue = Org.Mockito.Mockito.Spy(logValue); FilePath errorFile = new FilePath((new Path(srcFilePath, "stderr")).ToString()); Org.Mockito.Mockito.DoThrow(new IOException("Mock can not open FileInputStream")) .When(spyLogValue).SecureOpenFile(errorFile); logWriter.Append(logKey, spyLogValue); logWriter.Close(); // make sure permission are correct on the file FileStatus fsStatus = fs.GetFileStatus(remoteAppLogFile); NUnit.Framework.Assert.AreEqual("permissions on log aggregation file are wrong", FsPermission.CreateImmutable((short)0x1a0), fsStatus.GetPermission()); AggregatedLogFormat.LogReader logReader = new AggregatedLogFormat.LogReader(conf, remoteAppLogFile); AggregatedLogFormat.LogKey rLogKey = new AggregatedLogFormat.LogKey(); DataInputStream dis = logReader.Next(rLogKey); TextWriter writer = new StringWriter(); if (logUploadedTime) { AggregatedLogFormat.LogReader.ReadAcontainerLogs(dis, writer, Runtime.CurrentTimeMillis ()); } else { AggregatedLogFormat.LogReader.ReadAcontainerLogs(dis, writer); } // We should only do the log aggregation for stdout. // Since we could not open the fileInputStream for stderr, this file is not // aggregated. string s = writer.ToString(); int expectedLength = "LogType:stdout".Length + (logUploadedTime ? ("\nLog Upload Time:" + Times.Format(Runtime.CurrentTimeMillis())).Length : 0) + ("\nLogLength:" + numChars ).Length + "\nLog Contents:\n".Length + numChars + "\n".Length + "End of LogType:stdout\n" .Length; NUnit.Framework.Assert.IsTrue("LogType not matched", s.Contains("LogType:stdout") ); NUnit.Framework.Assert.IsTrue("log file:stderr should not be aggregated.", !s.Contains ("LogType:stderr")); NUnit.Framework.Assert.IsTrue("log file:logs should not be aggregated.", !s.Contains ("LogType:logs")); NUnit.Framework.Assert.IsTrue("LogLength not matched", s.Contains("LogLength:" + numChars)); NUnit.Framework.Assert.IsTrue("Log Contents not matched", s.Contains("Log Contents" )); StringBuilder sb = new StringBuilder(); for (int i = 0; i < numChars; i++) { sb.Append(filler); } string expectedContent = sb.ToString(); NUnit.Framework.Assert.IsTrue("Log content incorrect", s.Contains(expectedContent )); NUnit.Framework.Assert.AreEqual(expectedLength, s.Length); }
/// <summary>The content of this page is the attempts block</summary> /// <returns>AttemptsBlock.class</returns> protected override Type Content() { HistoryInfo info = new HistoryInfo(); Info("History Server").("BuildVersion", info.GetHadoopBuildVersion() + " on " + info .GetHadoopVersionBuiltOn()).("History Server started on", Times.Format(info.GetStartedOn ())); return(typeof(InfoBlock)); }