public AMAttemptInfo(AMInfo amInfo, string jobId, string user) { this.nodeHttpAddress = string.Empty; this.nodeId = string.Empty; string nmHost = amInfo.GetNodeManagerHost(); int nmHttpPort = amInfo.GetNodeManagerHttpPort(); int nmPort = amInfo.GetNodeManagerPort(); if (nmHost != null) { this.nodeHttpAddress = nmHost + ":" + nmHttpPort; NodeId nodeId = NodeId.NewInstance(nmHost, nmPort); this.nodeId = nodeId.ToString(); } this.id = amInfo.GetAppAttemptId().GetAttemptId(); this.startTime = amInfo.GetStartTime(); this.containerId = string.Empty; this.logsLink = string.Empty; ContainerId containerId = amInfo.GetContainerId(); if (containerId != null) { this.containerId = containerId.ToString(); this.logsLink = StringHelper.Join(MRWebAppUtil.GetYARNWebappScheme() + nodeHttpAddress , StringHelper.Ujoin("node", "containerlogs", this.containerId, user)); } }
/// <exception cref="System.Exception"/> protected override void ServiceInit(Configuration conf) { Configuration config = new YarnConfiguration(conf); config.SetBoolean(Dispatcher.DispatcherExitOnErrorKey, true); // This is required for WebApps to use https if enabled. MRWebAppUtil.Initialize(GetConfig()); try { DoSecureLogin(conf); } catch (IOException ie) { throw new YarnRuntimeException("History Server Failed to login", ie); } jobHistoryService = new JobHistory(); historyContext = (HistoryContext)jobHistoryService; stateStore = CreateStateStore(conf); this.jhsDTSecretManager = CreateJHSSecretManager(conf, stateStore); clientService = CreateHistoryClientService(); aggLogDelService = new AggregatedLogDeletionService(); hsAdminServer = new HSAdminServer(aggLogDelService, jobHistoryService); AddService(stateStore); AddService(new JobHistoryServer.HistoryServerSecretManagerService(this)); AddService(jobHistoryService); AddService(clientService); AddService(aggLogDelService); AddService(hsAdminServer); base.ServiceInit(config); }
/// <summary>Gets the URI to access the given application on MapReduce history server /// </summary> /// <param name="id">the ID for which a URI is returned</param> /// <returns>the tracking URI</returns> /// <exception cref="Sharpen.URISyntaxException"/> public override URI GetTrackingUri(ApplicationId id) { string jobSuffix = id.ToString().ReplaceFirst("^application_", "job_"); string historyServerAddress = MRWebAppUtil.GetJHSWebappURLWithScheme(GetConf()); return(new URI(historyServerAddress + "/jobhistory/job/" + jobSuffix)); }
protected internal AppController(Org.Apache.Hadoop.Mapreduce.V2.App.Webapp.App app , Configuration conf, Controller.RequestContext ctx, string title) : base(ctx) { this.app = app; Set(AppId, app.context.GetApplicationID().ToString()); Set(RmWeb, Joiner.Join(MRWebAppUtil.GetYARNWebappScheme(), WebAppUtils.GetResolvedRMWebAppURLWithoutScheme (conf, MRWebAppUtil.GetYARNHttpPolicy()))); }
/// <exception cref="System.Exception"/> protected override void ServiceStart() { lock (this) { try { if (!this.GetConfig().GetBoolean(JHAdminConfig.MrHistoryMiniclusterFixedPorts, JHAdminConfig .DefaultMrHistoryMiniclusterFixedPorts)) { string hostname = MiniYARNCluster.GetHostname(); // pick free random ports. this.GetConfig().Set(JHAdminConfig.MrHistoryAddress, hostname + ":0"); MRWebAppUtil.SetJHSWebappURLWithoutScheme(this.GetConfig(), hostname + ":0"); this.GetConfig().Set(JHAdminConfig.JhsAdminAddress, hostname + ":0"); } this._enclosing.historyServer = new JobHistoryServer(); this._enclosing.historyServer.Init(this.GetConfig()); new _Thread_212(this).Start(); while (!this.jhsStarted) { MiniMRYarnCluster.Log.Info("Waiting for HistoryServer to start..."); Sharpen.Thread.Sleep(1500); } //TODO Add a timeout. State.STOPPED check ? if (this._enclosing.historyServer.GetServiceState() != Service.STATE.Started) { throw new IOException("HistoryServer failed to start"); } base.ServiceStart(); } catch (Exception t) { throw new YarnRuntimeException(t); } //need to do this because historyServer.init creates a new Configuration this.GetConfig().Set(JHAdminConfig.MrHistoryAddress, this._enclosing.historyServer .GetConfig().Get(JHAdminConfig.MrHistoryAddress)); MRWebAppUtil.SetJHSWebappURLWithoutScheme(this.GetConfig(), MRWebAppUtil.GetJHSWebappURLWithoutScheme (this._enclosing.historyServer.GetConfig())); MiniMRYarnCluster.Log.Info("MiniMRYARN ResourceManager address: " + this.GetConfig ().Get(YarnConfiguration.RmAddress)); MiniMRYarnCluster.Log.Info("MiniMRYARN ResourceManager web address: " + WebAppUtils .GetRMWebAppURLWithoutScheme(this.GetConfig())); MiniMRYarnCluster.Log.Info("MiniMRYARN HistoryServer address: " + this.GetConfig( ).Get(JHAdminConfig.MrHistoryAddress)); MiniMRYarnCluster.Log.Info("MiniMRYARN HistoryServer web address: " + MiniMRYarnCluster .GetResolvedMRHistoryWebAppURLWithoutScheme(this.GetConfig(), MRWebAppUtil.GetJHSHttpPolicy () == HttpConfig.Policy.HttpsOnly)); } }
protected internal virtual void InitializeWebApp(Configuration conf) { webApp = new HsWebApp(history); IPEndPoint bindAddress = MRWebAppUtil.GetJHSWebBindAddress(conf); // NOTE: there should be a .at(InetSocketAddress) WebApps.$for <Org.Apache.Hadoop.Mapreduce.V2.HS.HistoryClientService>("jobhistory" , this, "ws").With(conf).WithHttpSpnegoKeytabKey(JHAdminConfig.MrWebappSpnegoKeytabFileKey ).WithHttpSpnegoPrincipalKey(JHAdminConfig.MrWebappSpnegoUserNameKey).At(NetUtils .GetHostPortString(bindAddress)).Start(webApp); string connectHost = MRWebAppUtil.GetJHSWebappURLWithoutScheme(conf).Split(":")[0 ]; MRWebAppUtil.SetJHSWebappURLWithoutScheme(conf, connectHost + ":" + webApp.GetListenerAddress ().Port); }
protected internal virtual void Register() { //Register IPEndPoint serviceAddr = null; if (clientService != null) { serviceAddr = clientService.GetBindAddress(); } try { RegisterApplicationMasterRequest request = recordFactory.NewRecordInstance <RegisterApplicationMasterRequest >(); if (serviceAddr != null) { request.SetHost(serviceAddr.GetHostName()); request.SetRpcPort(serviceAddr.Port); request.SetTrackingUrl(MRWebAppUtil.GetAMWebappScheme(GetConfig()) + serviceAddr. GetHostName() + ":" + clientService.GetHttpPort()); } RegisterApplicationMasterResponse response = scheduler.RegisterApplicationMaster( request); isApplicationMasterRegistered = true; maxContainerCapability = response.GetMaximumResourceCapability(); this.context.GetClusterInfo().SetMaxContainerCapability(maxContainerCapability); if (UserGroupInformation.IsSecurityEnabled()) { SetClientToAMToken(response.GetClientToAMTokenMasterKey()); } this.applicationACLs = response.GetApplicationACLs(); Log.Info("maxContainerCapability: " + maxContainerCapability); string queue = response.GetQueue(); Log.Info("queue: " + queue); job.SetQueueName(queue); Sharpen.Collections.AddAll(this.schedulerResourceTypes, response.GetSchedulerResourceTypes ()); } catch (Exception are) { Log.Error("Exception while registering", are); throw new YarnRuntimeException(are); } }
private void ConstructJobReport() { report = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <JobReport>(); report.SetJobId(jobId); report.SetJobState(JobState.ValueOf(jobInfo.GetJobStatus())); report.SetSubmitTime(jobInfo.GetSubmitTime()); report.SetStartTime(jobInfo.GetLaunchTime()); report.SetFinishTime(jobInfo.GetFinishTime()); report.SetJobName(jobInfo.GetJobname()); report.SetUser(jobInfo.GetUsername()); if (GetTotalMaps() == 0) { report.SetMapProgress(1.0f); } else { report.SetMapProgress((float)GetCompletedMaps() / GetTotalMaps()); } if (GetTotalReduces() == 0) { report.SetReduceProgress(1.0f); } else { report.SetReduceProgress((float)GetCompletedReduces() / GetTotalReduces()); } report.SetJobFile(GetConfFile().ToString()); string historyUrl = "N/A"; try { historyUrl = MRWebAppUtil.GetApplicationWebURLOnJHSWithoutScheme(conf, jobId.GetAppId ()); } catch (UnknownHostException) { } //Ignore. report.SetTrackingUrl(historyUrl); report.SetAMInfos(GetAMInfos()); report.SetIsUber(IsUber()); }
protected override void Render(HtmlBlock.Block html) { if (!IsValidRequest()) { html.H2($(Title)); return; } Hamlet.TBODY <Hamlet.TABLE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > tbody = html .Table("#attempts").Thead().Tr().Th(".id", "Attempt").Th(".progress", "Progress" ).Th(".state", "State").Th(".status", "Status").Th(".node", "Node").Th(".logs", "Logs").Th(".tsh", "Started").Th(".tsh", "Finished").Th(".tsh", "Elapsed").Th(".note" , "Note").().().Tbody(); // Write all the data into a JavaScript array of arrays for JQuery // DataTables to display StringBuilder attemptsTableData = new StringBuilder("[\n"); foreach (TaskAttempt attempt in GetTaskAttempts()) { TaskAttemptInfo ta = new TaskAttemptInfo(attempt, true); string progress = StringHelper.Percent(ta.GetProgress() / 100); string nodeHttpAddr = ta.GetNode(); string diag = ta.GetNote() == null ? string.Empty : ta.GetNote(); attemptsTableData.Append("[\"").Append(ta.GetId()).Append("\",\"").Append(progress ).Append("\",\"").Append(ta.GetState().ToString()).Append("\",\"").Append(StringEscapeUtils .EscapeJavaScript(StringEscapeUtils.EscapeHtml(ta.GetStatus()))).Append("\",\"") .Append(nodeHttpAddr == null ? "N/A" : "<a class='nodelink' href='" + MRWebAppUtil .GetYARNWebappScheme() + nodeHttpAddr + "'>" + nodeHttpAddr + "</a>").Append("\",\"" ).Append(ta.GetAssignedContainerId() == null ? "N/A" : "<a class='logslink' href='" + Url(MRWebAppUtil.GetYARNWebappScheme(), nodeHttpAddr, "node", "containerlogs" , ta.GetAssignedContainerIdStr(), app.GetJob().GetUserName()) + "'>logs</a>").Append ("\",\"").Append(ta.GetStartTime()).Append("\",\"").Append(ta.GetFinishTime()).Append ("\",\"").Append(ta.GetElapsedTime()).Append("\",\"").Append(StringEscapeUtils.EscapeJavaScript (StringEscapeUtils.EscapeHtml(diag))).Append("\"],\n"); } //Remove the last comma and close off the array of arrays if (attemptsTableData[attemptsTableData.Length - 2] == ',') { attemptsTableData.Delete(attemptsTableData.Length - 2, attemptsTableData.Length - 1); } attemptsTableData.Append("]"); html.Script().$type("text/javascript").("var attemptsTableData=" + attemptsTableData ).(); tbody.().(); }
protected override void Render(HtmlBlock.Block html) { string rmweb = $(AMParams.RmWeb); Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> nav = html.Div("#nav").H3 ("Cluster").Ul().Li().A(Url(rmweb, "cluster", "cluster"), "About").().Li().A(Url (rmweb, "cluster", "apps"), "Applications").().Li().A(Url(rmweb, "cluster", "scheduler" ), "Scheduler").().().H3("Application").Ul().Li().A(Url("app/info"), "About").() .Li().A(Url("app"), "Jobs").().(); if (app.GetJob() != null) { string jobid = MRApps.ToString(app.GetJob().GetID()); IList <AMInfo> amInfos = app.GetJob().GetAMInfos(); AMInfo thisAmInfo = amInfos[amInfos.Count - 1]; string nodeHttpAddress = thisAmInfo.GetNodeManagerHost() + ":" + thisAmInfo.GetNodeManagerHttpPort (); nav.H3("Job").Ul().Li().A(Url("job", jobid), "Overview").().Li().A(Url("jobcounters" , jobid), "Counters").().Li().A(Url("conf", jobid), "Configuration").().Li().A(Url ("tasks", jobid, "m"), "Map tasks").().Li().A(Url("tasks", jobid, "r"), "Reduce tasks" ).().Li().A(".logslink", Url(MRWebAppUtil.GetYARNWebappScheme(), nodeHttpAddress , "node", "containerlogs", thisAmInfo.GetContainerId().ToString(), app.GetJob(). GetUserName()), "AM Logs").().(); if (app.GetTask() != null) { string taskid = MRApps.ToString(app.GetTask().GetID()); nav.H3("Task").Ul().Li().A(Url("task", taskid), "Task Overview").().Li().A(Url("taskcounters" , taskid), "Counters").().(); } } nav.H3("Tools").Ul().Li().A("/conf", "Configuration").().Li().A("/logs", "Local logs" ).().Li().A("/stacks", "Server stacks").().Li().A("/jmx?qry=Hadoop:*", "Server metrics" ).().().(); }
protected override void Render(HtmlBlock.Block html) { if (!IsValidRequest()) { html.H2($(Title)); return; } TaskType type = null; string symbol = $(AMParams.TaskType); if (!symbol.IsEmpty()) { type = MRApps.TaskType(symbol); } else { type = app.GetTask().GetType(); } Hamlet.TR <Hamlet.THEAD <Hamlet.TABLE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > > headRow = html.Table("#attempts").Thead().Tr(); headRow.Th(".id", "Attempt").Th(".state", "State").Th(".status", "Status").Th(".node" , "Node").Th(".logs", "Logs").Th(".tsh", "Start Time"); if (type == TaskType.Reduce) { headRow.Th("Shuffle Finish Time"); headRow.Th("Merge Finish Time"); } headRow.Th("Finish Time"); //Attempt if (type == TaskType.Reduce) { headRow.Th("Elapsed Time Shuffle"); //Attempt headRow.Th("Elapsed Time Merge"); //Attempt headRow.Th("Elapsed Time Reduce"); } //Attempt headRow.Th("Elapsed Time").Th(".note", "Note"); Hamlet.TBODY <Hamlet.TABLE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > tbody = headRow .().().Tbody(); // Write all the data into a JavaScript array of arrays for JQuery // DataTables to display StringBuilder attemptsTableData = new StringBuilder("[\n"); foreach (TaskAttempt attempt in GetTaskAttempts()) { TaskAttemptInfo ta = new TaskAttemptInfo(attempt, false); string taid = ta.GetId(); string nodeHttpAddr = ta.GetNode(); string containerIdString = ta.GetAssignedContainerIdStr(); string nodeIdString = attempt.GetAssignedContainerMgrAddress(); string nodeRackName = ta.GetRack(); long attemptStartTime = ta.GetStartTime(); long shuffleFinishTime = -1; long sortFinishTime = -1; long attemptFinishTime = ta.GetFinishTime(); long elapsedShuffleTime = -1; long elapsedSortTime = -1; long elapsedReduceTime = -1; if (type == TaskType.Reduce) { shuffleFinishTime = attempt.GetShuffleFinishTime(); sortFinishTime = attempt.GetSortFinishTime(); elapsedShuffleTime = Times.Elapsed(attemptStartTime, shuffleFinishTime, false); elapsedSortTime = Times.Elapsed(shuffleFinishTime, sortFinishTime, false); elapsedReduceTime = Times.Elapsed(sortFinishTime, attemptFinishTime, false); } long attemptElapsed = Times.Elapsed(attemptStartTime, attemptFinishTime, false); int sortId = attempt.GetID().GetId() + (attempt.GetID().GetTaskId().GetId() * 10000 ); attemptsTableData.Append("[\"").Append(sortId + " ").Append(taid).Append("\",\"") .Append(ta.GetState()).Append("\",\"").Append(StringEscapeUtils.EscapeJavaScript (StringEscapeUtils.EscapeHtml(ta.GetStatus()))).Append("\",\"").Append("<a class='nodelink' href='" + MRWebAppUtil.GetYARNWebappScheme() + nodeHttpAddr + "'>").Append(nodeRackName + "/" + nodeHttpAddr + "</a>\",\"").Append("<a class='logslink' href='").Append (Url("logs", nodeIdString, containerIdString, taid, app.GetJob().GetUserName())) .Append("'>logs</a>\",\"").Append(attemptStartTime).Append("\",\""); if (type == TaskType.Reduce) { attemptsTableData.Append(shuffleFinishTime).Append("\",\"").Append(sortFinishTime ).Append("\",\""); } attemptsTableData.Append(attemptFinishTime).Append("\",\""); if (type == TaskType.Reduce) { attemptsTableData.Append(elapsedShuffleTime).Append("\",\"").Append(elapsedSortTime ).Append("\",\"").Append(elapsedReduceTime).Append("\",\""); } attemptsTableData.Append(attemptElapsed).Append("\",\"").Append(StringEscapeUtils .EscapeJavaScript(StringEscapeUtils.EscapeHtml(ta.GetNote()))).Append("\"],\n"); } //Remove the last comma and close off the array of arrays if (attemptsTableData[attemptsTableData.Length - 2] == ',') { attemptsTableData.Delete(attemptsTableData.Length - 2, attemptsTableData.Length - 1); } attemptsTableData.Append("]"); html.Script().$type("text/javascript").("var attemptsTableData=" + attemptsTableData ).(); Hamlet.TR <Hamlet.TFOOT <Hamlet.TABLE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > > footRow = tbody.().Tfoot().Tr(); footRow.Th().Input("search_init").$type(HamletSpec.InputType.text).$name("attempt_name" ).$value("Attempt").().().Th().Input("search_init").$type(HamletSpec.InputType.text ).$name("attempt_state").$value("State").().().Th().Input("search_init").$type(HamletSpec.InputType .text).$name("attempt_status").$value("Status").().().Th().Input("search_init"). $type(HamletSpec.InputType.text).$name("attempt_node").$value("Node").().().Th() .Input("search_init").$type(HamletSpec.InputType.text).$name("attempt_node").$value ("Logs").().().Th().Input("search_init").$type(HamletSpec.InputType.text).$name( "attempt_start_time").$value("Start Time").().(); if (type == TaskType.Reduce) { footRow.Th().Input("search_init").$type(HamletSpec.InputType.text).$name("shuffle_time" ).$value("Shuffle Time").().(); footRow.Th().Input("search_init").$type(HamletSpec.InputType.text).$name("merge_time" ).$value("Merge Time").().(); } footRow.Th().Input("search_init").$type(HamletSpec.InputType.text).$name("attempt_finish" ).$value("Finish Time").().(); if (type == TaskType.Reduce) { footRow.Th().Input("search_init").$type(HamletSpec.InputType.text).$name("elapsed_shuffle_time" ).$value("Elapsed Shuffle Time").().(); footRow.Th().Input("search_init").$type(HamletSpec.InputType.text).$name("elapsed_merge_time" ).$value("Elapsed Merge Time").().(); footRow.Th().Input("search_init").$type(HamletSpec.InputType.text).$name("elapsed_reduce_time" ).$value("Elapsed Reduce Time").().(); } footRow.Th().Input("search_init").$type(HamletSpec.InputType.text).$name("attempt_elapsed" ).$value("Elapsed Time").().().Th().Input("search_init").$type(HamletSpec.InputType .text).$name("note").$value("Note").().(); footRow.().().(); }
/* * (non-Javadoc) * @see org.apache.hadoop.yarn.webapp.view.HtmlBlock#render(org.apache.hadoop.yarn.webapp.view.HtmlBlock.Block) */ protected override void Render(HtmlBlock.Block html) { string jid = $(AMParams.JobId); if (jid.IsEmpty()) { html.P().("Sorry, can't do anything without a JobID.").(); return; } JobId jobID = MRApps.ToJobID(jid); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job j = appContext.GetJob(jobID); if (j == null) { html.P().("Sorry, ", jid, " not found.").(); return; } IList <AMInfo> amInfos = j.GetAMInfos(); JobInfo job = new JobInfo(j); ResponseInfo infoBlock = Info("Job Overview").("Job Name:", job.GetName()).("User Name:" , job.GetUserName()).("Queue:", job.GetQueueName()).("State:", job.GetState()).( "Uberized:", job.IsUber()).("Submitted:", Sharpen.Extensions.CreateDate(job.GetSubmitTime ())).("Started:", Sharpen.Extensions.CreateDate(job.GetStartTime())).("Finished:" , Sharpen.Extensions.CreateDate(job.GetFinishTime())).("Elapsed:", StringUtils.FormatTime (Times.Elapsed(job.GetStartTime(), job.GetFinishTime(), false))); string amString = amInfos.Count == 1 ? "ApplicationMaster" : "ApplicationMasters"; // todo - switch to use JobInfo IList <string> diagnostics = j.GetDiagnostics(); if (diagnostics != null && !diagnostics.IsEmpty()) { StringBuilder b = new StringBuilder(); foreach (string diag in diagnostics) { b.Append(diag); } infoBlock.("Diagnostics:", b.ToString()); } if (job.GetNumMaps() > 0) { infoBlock.("Average Map Time", StringUtils.FormatTime(job.GetAvgMapTime())); } if (job.GetNumReduces() > 0) { infoBlock.("Average Shuffle Time", StringUtils.FormatTime(job.GetAvgShuffleTime() )); infoBlock.("Average Merge Time", StringUtils.FormatTime(job.GetAvgMergeTime())); infoBlock.("Average Reduce Time", StringUtils.FormatTime(job.GetAvgReduceTime())); } foreach (ConfEntryInfo entry in job.GetAcls()) { infoBlock.("ACL " + entry.GetName() + ":", entry.GetValue()); } Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> div = html.(typeof(InfoBlock )).Div(JQueryUI.InfoWrap); // MRAppMasters Table Hamlet.TABLE <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > table = div .Table("#job"); table.Tr().Th(amString).().Tr().Th(JQueryUI.Th, "Attempt Number").Th(JQueryUI.Th, "Start Time").Th(JQueryUI.Th, "Node").Th(JQueryUI.Th, "Logs").(); bool odd = false; foreach (AMInfo amInfo in amInfos) { AMAttemptInfo attempt = new AMAttemptInfo(amInfo, job.GetId(), job.GetUserName(), string.Empty, string.Empty); table.Tr((odd = !odd) ? JQueryUI.Odd : JQueryUI.Even).Td(attempt.GetAttemptId().ToString ()).Td(Sharpen.Extensions.CreateDate(attempt.GetStartTime()).ToString()).Td().A( ".nodelink", Url(MRWebAppUtil.GetYARNWebappScheme(), attempt.GetNodeHttpAddress( )), attempt.GetNodeHttpAddress()).().Td().A(".logslink", Url(attempt.GetShortLogsLink ()), "logs").().(); } table.(); div.(); html.Div(JQueryUI.InfoWrap).Table("#job").Tr().Th(JQueryUI.Th, "Task Type").Th(JQueryUI .Th, "Total").Th(JQueryUI.Th, "Complete").().Tr(JQueryUI.Odd).Th().A(Url("tasks" , jid, "m"), "Map").().Td(job.GetMapsTotal().ToString().ToString()).Td(job.GetMapsCompleted ().ToString().ToString()).().Tr(JQueryUI.Even).Th().A(Url("tasks", jid, "r"), "Reduce" ).().Td(job.GetReducesTotal().ToString().ToString()).Td(job.GetReducesCompleted( ).ToString().ToString()).().().Table("#job").Tr().Th(JQueryUI.Th, "Attempt Type" ).Th(JQueryUI.Th, "Failed").Th(JQueryUI.Th, "Killed").Th(JQueryUI.Th, "Successful" ).().Tr(JQueryUI.Odd).Th("Maps").Td().A(Url("attempts", jid, "m", MRApps.TaskAttemptStateUI .Failed.ToString()), job.GetFailedMapAttempts().ToString()).().Td().A(Url("attempts" , jid, "m", MRApps.TaskAttemptStateUI.Killed.ToString()), job.GetKilledMapAttempts ().ToString()).().Td().A(Url("attempts", jid, "m", MRApps.TaskAttemptStateUI.Successful .ToString()), job.GetSuccessfulMapAttempts().ToString()).().().Tr(JQueryUI.Even) .Th("Reduces").Td().A(Url("attempts", jid, "r", MRApps.TaskAttemptStateUI.Failed .ToString()), job.GetFailedReduceAttempts().ToString()).().Td().A(Url("attempts" , jid, "r", MRApps.TaskAttemptStateUI.Killed.ToString()), job.GetKilledReduceAttempts ().ToString()).().Td().A(Url("attempts", jid, "r", MRApps.TaskAttemptStateUI.Successful .ToString()), job.GetSuccessfulReduceAttempts().ToString()).().().().(); }
protected override void Render(HtmlBlock.Block html) { string jid = $(AMParams.JobId); if (jid.IsEmpty()) { html.P().("Sorry, can't do anything without a JobID.").(); return; } JobId jobID = MRApps.ToJobID(jid); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = appContext.GetJob(jobID); if (job == null) { html.P().("Sorry, ", jid, " not found.").(); return; } IList <AMInfo> amInfos = job.GetAMInfos(); string amString = amInfos.Count == 1 ? "ApplicationMaster" : "ApplicationMasters"; JobInfo jinfo = new JobInfo(job, true); Info("Job Overview").("Job Name:", jinfo.GetName()).("State:", jinfo.GetState()). ("Uberized:", jinfo.IsUberized()).("Started:", Sharpen.Extensions.CreateDate(jinfo .GetStartTime())).("Elapsed:", StringUtils.FormatTime(jinfo.GetElapsedTime())); Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> div = html.(typeof(InfoBlock )).Div(JQueryUI.InfoWrap); // MRAppMasters Table Hamlet.TABLE <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > table = div .Table("#job"); table.Tr().Th(amString).().Tr().Th(JQueryUI.Th, "Attempt Number").Th(JQueryUI.Th, "Start Time").Th(JQueryUI.Th, "Node").Th(JQueryUI.Th, "Logs").(); foreach (AMInfo amInfo in amInfos) { AMAttemptInfo attempt = new AMAttemptInfo(amInfo, jinfo.GetId(), jinfo.GetUserName ()); table.Tr().Td(attempt.GetAttemptId().ToString()).Td(Sharpen.Extensions.CreateDate (attempt.GetStartTime()).ToString()).Td().A(".nodelink", Url(MRWebAppUtil.GetYARNWebappScheme (), attempt.GetNodeHttpAddress()), attempt.GetNodeHttpAddress()).().Td().A(".logslink" , Url(attempt.GetLogsLink()), "logs").().(); } table.(); div.(); html.Div(JQueryUI.InfoWrap).Table("#job").Tr().Th(JQueryUI.Th, "Task Type").Th(JQueryUI .Th, "Progress").Th(JQueryUI.Th, "Total").Th(JQueryUI.Th, "Pending").Th(JQueryUI .Th, "Running").Th(JQueryUI.Th, "Complete").().Tr(JQueryUI.Odd).Th("Map").Td().Div (JQueryUI.Progressbar).$title(StringHelper.Join(jinfo.GetMapProgressPercent(), '%' )).Div(JQueryUI.ProgressbarValue).$style(StringHelper.Join("width:", jinfo.GetMapProgressPercent (), '%')).().().().Td().A(Url("tasks", jid, "m", "ALL"), jinfo.GetMapsTotal().ToString ()).().Td().A(Url("tasks", jid, "m", "PENDING"), jinfo.GetMapsPending().ToString ()).().Td().A(Url("tasks", jid, "m", "RUNNING"), jinfo.GetMapsRunning().ToString ()).().Td().A(Url("tasks", jid, "m", "COMPLETED"), jinfo.GetMapsCompleted().ToString ()).().().Tr(JQueryUI.Even).Th("Reduce").Td().Div(JQueryUI.Progressbar).$title(StringHelper.Join (jinfo.GetReduceProgressPercent(), '%')).Div(JQueryUI.ProgressbarValue).$style(StringHelper.Join ("width:", jinfo.GetReduceProgressPercent(), '%')).().().().Td().A(Url("tasks", jid, "r", "ALL"), jinfo.GetReducesTotal().ToString()).().Td().A(Url("tasks", jid , "r", "PENDING"), jinfo.GetReducesPending().ToString()).().Td().A(Url("tasks", jid, "r", "RUNNING"), jinfo.GetReducesRunning().ToString()).().Td().A(Url("tasks" , jid, "r", "COMPLETED"), jinfo.GetReducesCompleted().ToString()).().().().Table ("#job").Tr().Th(JQueryUI.Th, "Attempt Type").Th(JQueryUI.Th, "New").Th(JQueryUI .Th, "Running").Th(JQueryUI.Th, "Failed").Th(JQueryUI.Th, "Killed").Th(JQueryUI. Th, "Successful").().Tr(JQueryUI.Odd).Th("Maps").Td().A(Url("attempts", jid, "m" , MRApps.TaskAttemptStateUI.New.ToString()), jinfo.GetNewMapAttempts().ToString( )).().Td().A(Url("attempts", jid, "m", MRApps.TaskAttemptStateUI.Running.ToString ()), jinfo.GetRunningMapAttempts().ToString()).().Td().A(Url("attempts", jid, "m" , MRApps.TaskAttemptStateUI.Failed.ToString()), jinfo.GetFailedMapAttempts().ToString ()).().Td().A(Url("attempts", jid, "m", MRApps.TaskAttemptStateUI.Killed.ToString ()), jinfo.GetKilledMapAttempts().ToString()).().Td().A(Url("attempts", jid, "m" , MRApps.TaskAttemptStateUI.Successful.ToString()), jinfo.GetSuccessfulMapAttempts ().ToString()).().().Tr(JQueryUI.Even).Th("Reduces").Td().A(Url("attempts", jid, "r", MRApps.TaskAttemptStateUI.New.ToString()), jinfo.GetNewReduceAttempts().ToString ()).().Td().A(Url("attempts", jid, "r", MRApps.TaskAttemptStateUI.Running.ToString ()), jinfo.GetRunningReduceAttempts().ToString()).().Td().A(Url("attempts", jid, "r", MRApps.TaskAttemptStateUI.Failed.ToString()), jinfo.GetFailedReduceAttempts ().ToString()).().Td().A(Url("attempts", jid, "r", MRApps.TaskAttemptStateUI.Killed .ToString()), jinfo.GetKilledReduceAttempts().ToString()).().Td().A(Url("attempts" , jid, "r", MRApps.TaskAttemptStateUI.Successful.ToString()), jinfo.GetSuccessfulReduceAttempts ().ToString()).().().().(); }
protected internal virtual void DoUnregistration() { FinalApplicationStatus finishState = FinalApplicationStatus.Undefined; JobImpl jobImpl = (JobImpl)job; if (jobImpl.GetInternalState() == JobStateInternal.Succeeded) { finishState = FinalApplicationStatus.Succeeded; } else { if (jobImpl.GetInternalState() == JobStateInternal.Killed || (jobImpl.GetInternalState () == JobStateInternal.Running && isSignalled)) { finishState = FinalApplicationStatus.Killed; } else { if (jobImpl.GetInternalState() == JobStateInternal.Failed || jobImpl.GetInternalState () == JobStateInternal.Error) { finishState = FinalApplicationStatus.Failed; } } } StringBuilder sb = new StringBuilder(); foreach (string s in job.GetDiagnostics()) { sb.Append(s).Append("\n"); } Log.Info("Setting job diagnostics to " + sb.ToString()); string historyUrl = MRWebAppUtil.GetApplicationWebURLOnJHSWithScheme(GetConfig(), context.GetApplicationID()); Log.Info("History url is " + historyUrl); FinishApplicationMasterRequest request = FinishApplicationMasterRequest.NewInstance (finishState, sb.ToString(), historyUrl); try { while (true) { FinishApplicationMasterResponse response = scheduler.FinishApplicationMaster(request ); if (response.GetIsUnregistered()) { // When excepting ClientService, other services are already stopped, // it is safe to let clients know the final states. ClientService // should wait for some time so clients have enough time to know the // final states. MRAppMaster.RunningAppContext raContext = (MRAppMaster.RunningAppContext)context; raContext.MarkSuccessfulUnregistration(); break; } Log.Info("Waiting for application to be successfully unregistered."); Sharpen.Thread.Sleep(rmPollInterval); } } catch (ApplicationMasterNotRegisteredException) { // RM might have restarted or failed over and so lost the fact that AM had // registered before. Register(); DoUnregistration(); } }