public AMAttemptInfo(AMInfo amInfo, string jobId, string user)
{
this.nodeHttpAddress = string.Empty;
this.nodeId = string.Empty;
string nmHost = amInfo.GetNodeManagerHost();
int nmHttpPort = amInfo.GetNodeManagerHttpPort();
int nmPort = amInfo.GetNodeManagerPort();
if (nmHost != null)
{
this.nodeHttpAddress = nmHost + ":" + nmHttpPort;
NodeId nodeId = NodeId.NewInstance(nmHost, nmPort);
this.nodeId = nodeId.ToString();
}
this.id = amInfo.GetAppAttemptId().GetAttemptId();
this.startTime = amInfo.GetStartTime();
this.containerId = string.Empty;
this.logsLink = string.Empty;
ContainerId containerId = amInfo.GetContainerId();
if (containerId != null)
{
this.containerId = containerId.ToString();
this.logsLink = StringHelper.Join(MRWebAppUtil.GetYARNWebappScheme() + nodeHttpAddress
, StringHelper.Ujoin("node", "containerlogs", this.containerId, user));
}
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
Configuration config = new YarnConfiguration(conf);
config.SetBoolean(Dispatcher.DispatcherExitOnErrorKey, true);
// This is required for WebApps to use https if enabled.
MRWebAppUtil.Initialize(GetConfig());
try
{
DoSecureLogin(conf);
}
catch (IOException ie)
{
throw new YarnRuntimeException("History Server Failed to login", ie);
}
jobHistoryService = new JobHistory();
historyContext = (HistoryContext)jobHistoryService;
stateStore = CreateStateStore(conf);
this.jhsDTSecretManager = CreateJHSSecretManager(conf, stateStore);
clientService = CreateHistoryClientService();
aggLogDelService = new AggregatedLogDeletionService();
hsAdminServer = new HSAdminServer(aggLogDelService, jobHistoryService);
AddService(stateStore);
AddService(new JobHistoryServer.HistoryServerSecretManagerService(this));
AddService(jobHistoryService);
AddService(clientService);
AddService(aggLogDelService);
AddService(hsAdminServer);
base.ServiceInit(config);
}
/// <summary>Gets the URI to access the given application on MapReduce history server
/// </summary>
/// <param name="id">the ID for which a URI is returned</param>
/// <returns>the tracking URI</returns>
/// <exception cref="Sharpen.URISyntaxException"/>
public override URI GetTrackingUri(ApplicationId id)
{
string jobSuffix = id.ToString().ReplaceFirst("^application_", "job_");
string historyServerAddress = MRWebAppUtil.GetJHSWebappURLWithScheme(GetConf());
return(new URI(historyServerAddress + "/jobhistory/job/" + jobSuffix));
}
protected internal AppController(Org.Apache.Hadoop.Mapreduce.V2.App.Webapp.App app
, Configuration conf, Controller.RequestContext ctx, string title)
: base(ctx)
{
this.app = app;
Set(AppId, app.context.GetApplicationID().ToString());
Set(RmWeb, Joiner.Join(MRWebAppUtil.GetYARNWebappScheme(), WebAppUtils.GetResolvedRMWebAppURLWithoutScheme
(conf, MRWebAppUtil.GetYARNHttpPolicy())));
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
lock (this)
{
try
{
if (!this.GetConfig().GetBoolean(JHAdminConfig.MrHistoryMiniclusterFixedPorts, JHAdminConfig
.DefaultMrHistoryMiniclusterFixedPorts))
{
string hostname = MiniYARNCluster.GetHostname();
// pick free random ports.
this.GetConfig().Set(JHAdminConfig.MrHistoryAddress, hostname + ":0");
MRWebAppUtil.SetJHSWebappURLWithoutScheme(this.GetConfig(), hostname + ":0");
this.GetConfig().Set(JHAdminConfig.JhsAdminAddress, hostname + ":0");
}
this._enclosing.historyServer = new JobHistoryServer();
this._enclosing.historyServer.Init(this.GetConfig());
new _Thread_212(this).Start();
while (!this.jhsStarted)
{
MiniMRYarnCluster.Log.Info("Waiting for HistoryServer to start...");
Sharpen.Thread.Sleep(1500);
}
//TODO Add a timeout. State.STOPPED check ?
if (this._enclosing.historyServer.GetServiceState() != Service.STATE.Started)
{
throw new IOException("HistoryServer failed to start");
}
base.ServiceStart();
}
catch (Exception t)
{
throw new YarnRuntimeException(t);
}
//need to do this because historyServer.init creates a new Configuration
this.GetConfig().Set(JHAdminConfig.MrHistoryAddress, this._enclosing.historyServer
.GetConfig().Get(JHAdminConfig.MrHistoryAddress));
MRWebAppUtil.SetJHSWebappURLWithoutScheme(this.GetConfig(), MRWebAppUtil.GetJHSWebappURLWithoutScheme
(this._enclosing.historyServer.GetConfig()));
MiniMRYarnCluster.Log.Info("MiniMRYARN ResourceManager address: " + this.GetConfig
().Get(YarnConfiguration.RmAddress));
MiniMRYarnCluster.Log.Info("MiniMRYARN ResourceManager web address: " + WebAppUtils
.GetRMWebAppURLWithoutScheme(this.GetConfig()));
MiniMRYarnCluster.Log.Info("MiniMRYARN HistoryServer address: " + this.GetConfig(
).Get(JHAdminConfig.MrHistoryAddress));
MiniMRYarnCluster.Log.Info("MiniMRYARN HistoryServer web address: " + MiniMRYarnCluster
.GetResolvedMRHistoryWebAppURLWithoutScheme(this.GetConfig(), MRWebAppUtil.GetJHSHttpPolicy
() == HttpConfig.Policy.HttpsOnly));
}
}
protected internal virtual void InitializeWebApp(Configuration conf)
{
webApp = new HsWebApp(history);
IPEndPoint bindAddress = MRWebAppUtil.GetJHSWebBindAddress(conf);
// NOTE: there should be a .at(InetSocketAddress)
WebApps.$for <Org.Apache.Hadoop.Mapreduce.V2.HS.HistoryClientService>("jobhistory"
, this, "ws").With(conf).WithHttpSpnegoKeytabKey(JHAdminConfig.MrWebappSpnegoKeytabFileKey
).WithHttpSpnegoPrincipalKey(JHAdminConfig.MrWebappSpnegoUserNameKey).At(NetUtils
.GetHostPortString(bindAddress)).Start(webApp);
string connectHost = MRWebAppUtil.GetJHSWebappURLWithoutScheme(conf).Split(":")[0
];
MRWebAppUtil.SetJHSWebappURLWithoutScheme(conf, connectHost + ":" + webApp.GetListenerAddress
().Port);
}
protected internal virtual void Register()
{
//Register
IPEndPoint serviceAddr = null;
if (clientService != null)
{
serviceAddr = clientService.GetBindAddress();
}
try
{
RegisterApplicationMasterRequest request = recordFactory.NewRecordInstance <RegisterApplicationMasterRequest
>();
if (serviceAddr != null)
{
request.SetHost(serviceAddr.GetHostName());
request.SetRpcPort(serviceAddr.Port);
request.SetTrackingUrl(MRWebAppUtil.GetAMWebappScheme(GetConfig()) + serviceAddr.
GetHostName() + ":" + clientService.GetHttpPort());
}
RegisterApplicationMasterResponse response = scheduler.RegisterApplicationMaster(
request);
isApplicationMasterRegistered = true;
maxContainerCapability = response.GetMaximumResourceCapability();
this.context.GetClusterInfo().SetMaxContainerCapability(maxContainerCapability);
if (UserGroupInformation.IsSecurityEnabled())
{
SetClientToAMToken(response.GetClientToAMTokenMasterKey());
}
this.applicationACLs = response.GetApplicationACLs();
Log.Info("maxContainerCapability: " + maxContainerCapability);
string queue = response.GetQueue();
Log.Info("queue: " + queue);
job.SetQueueName(queue);
Sharpen.Collections.AddAll(this.schedulerResourceTypes, response.GetSchedulerResourceTypes
());
}
catch (Exception are)
{
Log.Error("Exception while registering", are);
throw new YarnRuntimeException(are);
}
}
private void ConstructJobReport()
{
report = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <JobReport>();
report.SetJobId(jobId);
report.SetJobState(JobState.ValueOf(jobInfo.GetJobStatus()));
report.SetSubmitTime(jobInfo.GetSubmitTime());
report.SetStartTime(jobInfo.GetLaunchTime());
report.SetFinishTime(jobInfo.GetFinishTime());
report.SetJobName(jobInfo.GetJobname());
report.SetUser(jobInfo.GetUsername());
if (GetTotalMaps() == 0)
{
report.SetMapProgress(1.0f);
}
else
{
report.SetMapProgress((float)GetCompletedMaps() / GetTotalMaps());
}
if (GetTotalReduces() == 0)
{
report.SetReduceProgress(1.0f);
}
else
{
report.SetReduceProgress((float)GetCompletedReduces() / GetTotalReduces());
}
report.SetJobFile(GetConfFile().ToString());
string historyUrl = "N/A";
try
{
historyUrl = MRWebAppUtil.GetApplicationWebURLOnJHSWithoutScheme(conf, jobId.GetAppId
());
}
catch (UnknownHostException)
{
}
//Ignore.
report.SetTrackingUrl(historyUrl);
report.SetAMInfos(GetAMInfos());
report.SetIsUber(IsUber());
}
protected override void Render(HtmlBlock.Block html)
{
if (!IsValidRequest())
{
html.H2($(Title));
return;
}
Hamlet.TBODY <Hamlet.TABLE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > tbody = html
.Table("#attempts").Thead().Tr().Th(".id", "Attempt").Th(".progress", "Progress"
).Th(".state", "State").Th(".status", "Status").Th(".node", "Node").Th(".logs",
"Logs").Th(".tsh", "Started").Th(".tsh", "Finished").Th(".tsh", "Elapsed").Th(".note"
, "Note").().().Tbody();
// Write all the data into a JavaScript array of arrays for JQuery
// DataTables to display
StringBuilder attemptsTableData = new StringBuilder("[\n");
foreach (TaskAttempt attempt in GetTaskAttempts())
{
TaskAttemptInfo ta = new TaskAttemptInfo(attempt, true);
string progress = StringHelper.Percent(ta.GetProgress() / 100);
string nodeHttpAddr = ta.GetNode();
string diag = ta.GetNote() == null ? string.Empty : ta.GetNote();
attemptsTableData.Append("[\"").Append(ta.GetId()).Append("\",\"").Append(progress
).Append("\",\"").Append(ta.GetState().ToString()).Append("\",\"").Append(StringEscapeUtils
.EscapeJavaScript(StringEscapeUtils.EscapeHtml(ta.GetStatus()))).Append("\",\"")
.Append(nodeHttpAddr == null ? "N/A" : "<a class='nodelink' href='" + MRWebAppUtil
.GetYARNWebappScheme() + nodeHttpAddr + "'>" + nodeHttpAddr + "</a>").Append("\",\""
).Append(ta.GetAssignedContainerId() == null ? "N/A" : "<a class='logslink' href='"
+ Url(MRWebAppUtil.GetYARNWebappScheme(), nodeHttpAddr, "node", "containerlogs"
, ta.GetAssignedContainerIdStr(), app.GetJob().GetUserName()) + "'>logs</a>").Append
("\",\"").Append(ta.GetStartTime()).Append("\",\"").Append(ta.GetFinishTime()).Append
("\",\"").Append(ta.GetElapsedTime()).Append("\",\"").Append(StringEscapeUtils.EscapeJavaScript
(StringEscapeUtils.EscapeHtml(diag))).Append("\"],\n");
}
//Remove the last comma and close off the array of arrays
if (attemptsTableData[attemptsTableData.Length - 2] == ',')
{
attemptsTableData.Delete(attemptsTableData.Length - 2, attemptsTableData.Length -
1);
}
attemptsTableData.Append("]");
html.Script().$type("text/javascript").("var attemptsTableData=" + attemptsTableData
).();
tbody.().();
}
protected override void Render(HtmlBlock.Block html)
{
string rmweb = $(AMParams.RmWeb);
Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> nav = html.Div("#nav").H3
("Cluster").Ul().Li().A(Url(rmweb, "cluster", "cluster"), "About").().Li().A(Url
(rmweb, "cluster", "apps"), "Applications").().Li().A(Url(rmweb, "cluster", "scheduler"
), "Scheduler").().().H3("Application").Ul().Li().A(Url("app/info"), "About").()
.Li().A(Url("app"), "Jobs").().();
if (app.GetJob() != null)
{
string jobid = MRApps.ToString(app.GetJob().GetID());
IList <AMInfo> amInfos = app.GetJob().GetAMInfos();
AMInfo thisAmInfo = amInfos[amInfos.Count - 1];
string nodeHttpAddress = thisAmInfo.GetNodeManagerHost() + ":" + thisAmInfo.GetNodeManagerHttpPort
();
nav.H3("Job").Ul().Li().A(Url("job", jobid), "Overview").().Li().A(Url("jobcounters"
, jobid), "Counters").().Li().A(Url("conf", jobid), "Configuration").().Li().A(Url
("tasks", jobid, "m"), "Map tasks").().Li().A(Url("tasks", jobid, "r"), "Reduce tasks"
).().Li().A(".logslink", Url(MRWebAppUtil.GetYARNWebappScheme(), nodeHttpAddress
, "node", "containerlogs", thisAmInfo.GetContainerId().ToString(), app.GetJob().
GetUserName()), "AM Logs").().();
if (app.GetTask() != null)
{
string taskid = MRApps.ToString(app.GetTask().GetID());
nav.H3("Task").Ul().Li().A(Url("task", taskid), "Task Overview").().Li().A(Url("taskcounters"
, taskid), "Counters").().();
}
}
nav.H3("Tools").Ul().Li().A("/conf", "Configuration").().Li().A("/logs", "Local logs"
).().Li().A("/stacks", "Server stacks").().Li().A("/jmx?qry=Hadoop:*", "Server metrics"
).().().();
}
protected override void Render(HtmlBlock.Block html)
{
if (!IsValidRequest())
{
html.H2($(Title));
return;
}
TaskType type = null;
string symbol = $(AMParams.TaskType);
if (!symbol.IsEmpty())
{
type = MRApps.TaskType(symbol);
}
else
{
type = app.GetTask().GetType();
}
Hamlet.TR <Hamlet.THEAD <Hamlet.TABLE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> >
> headRow = html.Table("#attempts").Thead().Tr();
headRow.Th(".id", "Attempt").Th(".state", "State").Th(".status", "Status").Th(".node"
, "Node").Th(".logs", "Logs").Th(".tsh", "Start Time");
if (type == TaskType.Reduce)
{
headRow.Th("Shuffle Finish Time");
headRow.Th("Merge Finish Time");
}
headRow.Th("Finish Time");
//Attempt
if (type == TaskType.Reduce)
{
headRow.Th("Elapsed Time Shuffle");
//Attempt
headRow.Th("Elapsed Time Merge");
//Attempt
headRow.Th("Elapsed Time Reduce");
}
//Attempt
headRow.Th("Elapsed Time").Th(".note", "Note");
Hamlet.TBODY <Hamlet.TABLE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > tbody = headRow
.().().Tbody();
// Write all the data into a JavaScript array of arrays for JQuery
// DataTables to display
StringBuilder attemptsTableData = new StringBuilder("[\n");
foreach (TaskAttempt attempt in GetTaskAttempts())
{
TaskAttemptInfo ta = new TaskAttemptInfo(attempt, false);
string taid = ta.GetId();
string nodeHttpAddr = ta.GetNode();
string containerIdString = ta.GetAssignedContainerIdStr();
string nodeIdString = attempt.GetAssignedContainerMgrAddress();
string nodeRackName = ta.GetRack();
long attemptStartTime = ta.GetStartTime();
long shuffleFinishTime = -1;
long sortFinishTime = -1;
long attemptFinishTime = ta.GetFinishTime();
long elapsedShuffleTime = -1;
long elapsedSortTime = -1;
long elapsedReduceTime = -1;
if (type == TaskType.Reduce)
{
shuffleFinishTime = attempt.GetShuffleFinishTime();
sortFinishTime = attempt.GetSortFinishTime();
elapsedShuffleTime = Times.Elapsed(attemptStartTime, shuffleFinishTime, false);
elapsedSortTime = Times.Elapsed(shuffleFinishTime, sortFinishTime, false);
elapsedReduceTime = Times.Elapsed(sortFinishTime, attemptFinishTime, false);
}
long attemptElapsed = Times.Elapsed(attemptStartTime, attemptFinishTime, false);
int sortId = attempt.GetID().GetId() + (attempt.GetID().GetTaskId().GetId() * 10000
);
attemptsTableData.Append("[\"").Append(sortId + " ").Append(taid).Append("\",\"")
.Append(ta.GetState()).Append("\",\"").Append(StringEscapeUtils.EscapeJavaScript
(StringEscapeUtils.EscapeHtml(ta.GetStatus()))).Append("\",\"").Append("<a class='nodelink' href='"
+ MRWebAppUtil.GetYARNWebappScheme() + nodeHttpAddr + "'>").Append(nodeRackName
+ "/" + nodeHttpAddr + "</a>\",\"").Append("<a class='logslink' href='").Append
(Url("logs", nodeIdString, containerIdString, taid, app.GetJob().GetUserName()))
.Append("'>logs</a>\",\"").Append(attemptStartTime).Append("\",\"");
if (type == TaskType.Reduce)
{
attemptsTableData.Append(shuffleFinishTime).Append("\",\"").Append(sortFinishTime
).Append("\",\"");
}
attemptsTableData.Append(attemptFinishTime).Append("\",\"");
if (type == TaskType.Reduce)
{
attemptsTableData.Append(elapsedShuffleTime).Append("\",\"").Append(elapsedSortTime
).Append("\",\"").Append(elapsedReduceTime).Append("\",\"");
}
attemptsTableData.Append(attemptElapsed).Append("\",\"").Append(StringEscapeUtils
.EscapeJavaScript(StringEscapeUtils.EscapeHtml(ta.GetNote()))).Append("\"],\n");
}
//Remove the last comma and close off the array of arrays
if (attemptsTableData[attemptsTableData.Length - 2] == ',')
{
attemptsTableData.Delete(attemptsTableData.Length - 2, attemptsTableData.Length -
1);
}
attemptsTableData.Append("]");
html.Script().$type("text/javascript").("var attemptsTableData=" + attemptsTableData
).();
Hamlet.TR <Hamlet.TFOOT <Hamlet.TABLE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> >
> footRow = tbody.().Tfoot().Tr();
footRow.Th().Input("search_init").$type(HamletSpec.InputType.text).$name("attempt_name"
).$value("Attempt").().().Th().Input("search_init").$type(HamletSpec.InputType.text
).$name("attempt_state").$value("State").().().Th().Input("search_init").$type(HamletSpec.InputType
.text).$name("attempt_status").$value("Status").().().Th().Input("search_init").
$type(HamletSpec.InputType.text).$name("attempt_node").$value("Node").().().Th()
.Input("search_init").$type(HamletSpec.InputType.text).$name("attempt_node").$value
("Logs").().().Th().Input("search_init").$type(HamletSpec.InputType.text).$name(
"attempt_start_time").$value("Start Time").().();
if (type == TaskType.Reduce)
{
footRow.Th().Input("search_init").$type(HamletSpec.InputType.text).$name("shuffle_time"
).$value("Shuffle Time").().();
footRow.Th().Input("search_init").$type(HamletSpec.InputType.text).$name("merge_time"
).$value("Merge Time").().();
}
footRow.Th().Input("search_init").$type(HamletSpec.InputType.text).$name("attempt_finish"
).$value("Finish Time").().();
if (type == TaskType.Reduce)
{
footRow.Th().Input("search_init").$type(HamletSpec.InputType.text).$name("elapsed_shuffle_time"
).$value("Elapsed Shuffle Time").().();
footRow.Th().Input("search_init").$type(HamletSpec.InputType.text).$name("elapsed_merge_time"
).$value("Elapsed Merge Time").().();
footRow.Th().Input("search_init").$type(HamletSpec.InputType.text).$name("elapsed_reduce_time"
).$value("Elapsed Reduce Time").().();
}
footRow.Th().Input("search_init").$type(HamletSpec.InputType.text).$name("attempt_elapsed"
).$value("Elapsed Time").().().Th().Input("search_init").$type(HamletSpec.InputType
.text).$name("note").$value("Note").().();
footRow.().().();
}
/*
* (non-Javadoc)
* @see org.apache.hadoop.yarn.webapp.view.HtmlBlock#render(org.apache.hadoop.yarn.webapp.view.HtmlBlock.Block)
*/
protected override void Render(HtmlBlock.Block html)
{
string jid = $(AMParams.JobId);
if (jid.IsEmpty())
{
html.P().("Sorry, can't do anything without a JobID.").();
return;
}
JobId jobID = MRApps.ToJobID(jid);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job j = appContext.GetJob(jobID);
if (j == null)
{
html.P().("Sorry, ", jid, " not found.").();
return;
}
IList <AMInfo> amInfos = j.GetAMInfos();
JobInfo job = new JobInfo(j);
ResponseInfo infoBlock = Info("Job Overview").("Job Name:", job.GetName()).("User Name:"
, job.GetUserName()).("Queue:", job.GetQueueName()).("State:", job.GetState()).(
"Uberized:", job.IsUber()).("Submitted:", Sharpen.Extensions.CreateDate(job.GetSubmitTime
())).("Started:", Sharpen.Extensions.CreateDate(job.GetStartTime())).("Finished:"
, Sharpen.Extensions.CreateDate(job.GetFinishTime())).("Elapsed:", StringUtils.FormatTime
(Times.Elapsed(job.GetStartTime(), job.GetFinishTime(), false)));
string amString = amInfos.Count == 1 ? "ApplicationMaster" : "ApplicationMasters";
// todo - switch to use JobInfo
IList <string> diagnostics = j.GetDiagnostics();
if (diagnostics != null && !diagnostics.IsEmpty())
{
StringBuilder b = new StringBuilder();
foreach (string diag in diagnostics)
{
b.Append(diag);
}
infoBlock.("Diagnostics:", b.ToString());
}
if (job.GetNumMaps() > 0)
{
infoBlock.("Average Map Time", StringUtils.FormatTime(job.GetAvgMapTime()));
}
if (job.GetNumReduces() > 0)
{
infoBlock.("Average Shuffle Time", StringUtils.FormatTime(job.GetAvgShuffleTime()
));
infoBlock.("Average Merge Time", StringUtils.FormatTime(job.GetAvgMergeTime()));
infoBlock.("Average Reduce Time", StringUtils.FormatTime(job.GetAvgReduceTime()));
}
foreach (ConfEntryInfo entry in job.GetAcls())
{
infoBlock.("ACL " + entry.GetName() + ":", entry.GetValue());
}
Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> div = html.(typeof(InfoBlock
)).Div(JQueryUI.InfoWrap);
// MRAppMasters Table
Hamlet.TABLE <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > table = div
.Table("#job");
table.Tr().Th(amString).().Tr().Th(JQueryUI.Th, "Attempt Number").Th(JQueryUI.Th,
"Start Time").Th(JQueryUI.Th, "Node").Th(JQueryUI.Th, "Logs").();
bool odd = false;
foreach (AMInfo amInfo in amInfos)
{
AMAttemptInfo attempt = new AMAttemptInfo(amInfo, job.GetId(), job.GetUserName(),
string.Empty, string.Empty);
table.Tr((odd = !odd) ? JQueryUI.Odd : JQueryUI.Even).Td(attempt.GetAttemptId().ToString
()).Td(Sharpen.Extensions.CreateDate(attempt.GetStartTime()).ToString()).Td().A(
".nodelink", Url(MRWebAppUtil.GetYARNWebappScheme(), attempt.GetNodeHttpAddress(
)), attempt.GetNodeHttpAddress()).().Td().A(".logslink", Url(attempt.GetShortLogsLink
()), "logs").().();
}
table.();
div.();
html.Div(JQueryUI.InfoWrap).Table("#job").Tr().Th(JQueryUI.Th, "Task Type").Th(JQueryUI
.Th, "Total").Th(JQueryUI.Th, "Complete").().Tr(JQueryUI.Odd).Th().A(Url("tasks"
, jid, "m"), "Map").().Td(job.GetMapsTotal().ToString().ToString()).Td(job.GetMapsCompleted
().ToString().ToString()).().Tr(JQueryUI.Even).Th().A(Url("tasks", jid, "r"), "Reduce"
).().Td(job.GetReducesTotal().ToString().ToString()).Td(job.GetReducesCompleted(
).ToString().ToString()).().().Table("#job").Tr().Th(JQueryUI.Th, "Attempt Type"
).Th(JQueryUI.Th, "Failed").Th(JQueryUI.Th, "Killed").Th(JQueryUI.Th, "Successful"
).().Tr(JQueryUI.Odd).Th("Maps").Td().A(Url("attempts", jid, "m", MRApps.TaskAttemptStateUI
.Failed.ToString()), job.GetFailedMapAttempts().ToString()).().Td().A(Url("attempts"
, jid, "m", MRApps.TaskAttemptStateUI.Killed.ToString()), job.GetKilledMapAttempts
().ToString()).().Td().A(Url("attempts", jid, "m", MRApps.TaskAttemptStateUI.Successful
.ToString()), job.GetSuccessfulMapAttempts().ToString()).().().Tr(JQueryUI.Even)
.Th("Reduces").Td().A(Url("attempts", jid, "r", MRApps.TaskAttemptStateUI.Failed
.ToString()), job.GetFailedReduceAttempts().ToString()).().Td().A(Url("attempts"
, jid, "r", MRApps.TaskAttemptStateUI.Killed.ToString()), job.GetKilledReduceAttempts
().ToString()).().Td().A(Url("attempts", jid, "r", MRApps.TaskAttemptStateUI.Successful
.ToString()), job.GetSuccessfulReduceAttempts().ToString()).().().().();
}
protected override void Render(HtmlBlock.Block html)
{
string jid = $(AMParams.JobId);
if (jid.IsEmpty())
{
html.P().("Sorry, can't do anything without a JobID.").();
return;
}
JobId jobID = MRApps.ToJobID(jid);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = appContext.GetJob(jobID);
if (job == null)
{
html.P().("Sorry, ", jid, " not found.").();
return;
}
IList <AMInfo> amInfos = job.GetAMInfos();
string amString = amInfos.Count == 1 ? "ApplicationMaster" : "ApplicationMasters";
JobInfo jinfo = new JobInfo(job, true);
Info("Job Overview").("Job Name:", jinfo.GetName()).("State:", jinfo.GetState()).
("Uberized:", jinfo.IsUberized()).("Started:", Sharpen.Extensions.CreateDate(jinfo
.GetStartTime())).("Elapsed:", StringUtils.FormatTime(jinfo.GetElapsedTime()));
Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> div = html.(typeof(InfoBlock
)).Div(JQueryUI.InfoWrap);
// MRAppMasters Table
Hamlet.TABLE <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > table = div
.Table("#job");
table.Tr().Th(amString).().Tr().Th(JQueryUI.Th, "Attempt Number").Th(JQueryUI.Th,
"Start Time").Th(JQueryUI.Th, "Node").Th(JQueryUI.Th, "Logs").();
foreach (AMInfo amInfo in amInfos)
{
AMAttemptInfo attempt = new AMAttemptInfo(amInfo, jinfo.GetId(), jinfo.GetUserName
());
table.Tr().Td(attempt.GetAttemptId().ToString()).Td(Sharpen.Extensions.CreateDate
(attempt.GetStartTime()).ToString()).Td().A(".nodelink", Url(MRWebAppUtil.GetYARNWebappScheme
(), attempt.GetNodeHttpAddress()), attempt.GetNodeHttpAddress()).().Td().A(".logslink"
, Url(attempt.GetLogsLink()), "logs").().();
}
table.();
div.();
html.Div(JQueryUI.InfoWrap).Table("#job").Tr().Th(JQueryUI.Th, "Task Type").Th(JQueryUI
.Th, "Progress").Th(JQueryUI.Th, "Total").Th(JQueryUI.Th, "Pending").Th(JQueryUI
.Th, "Running").Th(JQueryUI.Th, "Complete").().Tr(JQueryUI.Odd).Th("Map").Td().Div
(JQueryUI.Progressbar).$title(StringHelper.Join(jinfo.GetMapProgressPercent(), '%'
)).Div(JQueryUI.ProgressbarValue).$style(StringHelper.Join("width:", jinfo.GetMapProgressPercent
(), '%')).().().().Td().A(Url("tasks", jid, "m", "ALL"), jinfo.GetMapsTotal().ToString
()).().Td().A(Url("tasks", jid, "m", "PENDING"), jinfo.GetMapsPending().ToString
()).().Td().A(Url("tasks", jid, "m", "RUNNING"), jinfo.GetMapsRunning().ToString
()).().Td().A(Url("tasks", jid, "m", "COMPLETED"), jinfo.GetMapsCompleted().ToString
()).().().Tr(JQueryUI.Even).Th("Reduce").Td().Div(JQueryUI.Progressbar).$title(StringHelper.Join
(jinfo.GetReduceProgressPercent(), '%')).Div(JQueryUI.ProgressbarValue).$style(StringHelper.Join
("width:", jinfo.GetReduceProgressPercent(), '%')).().().().Td().A(Url("tasks",
jid, "r", "ALL"), jinfo.GetReducesTotal().ToString()).().Td().A(Url("tasks", jid
, "r", "PENDING"), jinfo.GetReducesPending().ToString()).().Td().A(Url("tasks",
jid, "r", "RUNNING"), jinfo.GetReducesRunning().ToString()).().Td().A(Url("tasks"
, jid, "r", "COMPLETED"), jinfo.GetReducesCompleted().ToString()).().().().Table
("#job").Tr().Th(JQueryUI.Th, "Attempt Type").Th(JQueryUI.Th, "New").Th(JQueryUI
.Th, "Running").Th(JQueryUI.Th, "Failed").Th(JQueryUI.Th, "Killed").Th(JQueryUI.
Th, "Successful").().Tr(JQueryUI.Odd).Th("Maps").Td().A(Url("attempts", jid, "m"
, MRApps.TaskAttemptStateUI.New.ToString()), jinfo.GetNewMapAttempts().ToString(
)).().Td().A(Url("attempts", jid, "m", MRApps.TaskAttemptStateUI.Running.ToString
()), jinfo.GetRunningMapAttempts().ToString()).().Td().A(Url("attempts", jid, "m"
, MRApps.TaskAttemptStateUI.Failed.ToString()), jinfo.GetFailedMapAttempts().ToString
()).().Td().A(Url("attempts", jid, "m", MRApps.TaskAttemptStateUI.Killed.ToString
()), jinfo.GetKilledMapAttempts().ToString()).().Td().A(Url("attempts", jid, "m"
, MRApps.TaskAttemptStateUI.Successful.ToString()), jinfo.GetSuccessfulMapAttempts
().ToString()).().().Tr(JQueryUI.Even).Th("Reduces").Td().A(Url("attempts", jid,
"r", MRApps.TaskAttemptStateUI.New.ToString()), jinfo.GetNewReduceAttempts().ToString
()).().Td().A(Url("attempts", jid, "r", MRApps.TaskAttemptStateUI.Running.ToString
()), jinfo.GetRunningReduceAttempts().ToString()).().Td().A(Url("attempts", jid,
"r", MRApps.TaskAttemptStateUI.Failed.ToString()), jinfo.GetFailedReduceAttempts
().ToString()).().Td().A(Url("attempts", jid, "r", MRApps.TaskAttemptStateUI.Killed
.ToString()), jinfo.GetKilledReduceAttempts().ToString()).().Td().A(Url("attempts"
, jid, "r", MRApps.TaskAttemptStateUI.Successful.ToString()), jinfo.GetSuccessfulReduceAttempts
().ToString()).().().().();
}
protected internal virtual void DoUnregistration()
{
FinalApplicationStatus finishState = FinalApplicationStatus.Undefined;
JobImpl jobImpl = (JobImpl)job;
if (jobImpl.GetInternalState() == JobStateInternal.Succeeded)
{
finishState = FinalApplicationStatus.Succeeded;
}
else
{
if (jobImpl.GetInternalState() == JobStateInternal.Killed || (jobImpl.GetInternalState
() == JobStateInternal.Running && isSignalled))
{
finishState = FinalApplicationStatus.Killed;
}
else
{
if (jobImpl.GetInternalState() == JobStateInternal.Failed || jobImpl.GetInternalState
() == JobStateInternal.Error)
{
finishState = FinalApplicationStatus.Failed;
}
}
}
StringBuilder sb = new StringBuilder();
foreach (string s in job.GetDiagnostics())
{
sb.Append(s).Append("\n");
}
Log.Info("Setting job diagnostics to " + sb.ToString());
string historyUrl = MRWebAppUtil.GetApplicationWebURLOnJHSWithScheme(GetConfig(),
context.GetApplicationID());
Log.Info("History url is " + historyUrl);
FinishApplicationMasterRequest request = FinishApplicationMasterRequest.NewInstance
(finishState, sb.ToString(), historyUrl);
try
{
while (true)
{
FinishApplicationMasterResponse response = scheduler.FinishApplicationMaster(request
);
if (response.GetIsUnregistered())
{
// When excepting ClientService, other services are already stopped,
// it is safe to let clients know the final states. ClientService
// should wait for some time so clients have enough time to know the
// final states.
MRAppMaster.RunningAppContext raContext = (MRAppMaster.RunningAppContext)context;
raContext.MarkSuccessfulUnregistration();
break;
}
Log.Info("Waiting for application to be successfully unregistered.");
Sharpen.Thread.Sleep(rmPollInterval);
}
}
catch (ApplicationMasterNotRegisteredException)
{
// RM might have restarted or failed over and so lost the fact that AM had
// registered before.
Register();
DoUnregistration();
}
}
