public AMAttemptInfo(AMInfo amInfo, string jobId, string user, string host, string
pathPrefix)
{
this.nodeHttpAddress = string.Empty;
this.nodeId = string.Empty;
string nmHost = amInfo.GetNodeManagerHost();
int nmHttpPort = amInfo.GetNodeManagerHttpPort();
int nmPort = amInfo.GetNodeManagerPort();
if (nmHost != null)
{
this.nodeHttpAddress = nmHost + ":" + nmHttpPort;
NodeId nodeId = NodeId.NewInstance(nmHost, nmPort);
this.nodeId = nodeId.ToString();
}
this.id = amInfo.GetAppAttemptId().GetAttemptId();
this.startTime = amInfo.GetStartTime();
this.containerId = string.Empty;
this.logsLink = string.Empty;
this.shortLogsLink = string.Empty;
ContainerId containerId = amInfo.GetContainerId();
if (containerId != null)
{
this.containerId = containerId.ToString();
this.logsLink = StringHelper.Join(host, pathPrefix, StringHelper.Ujoin("logs", this
.nodeId, this.containerId, jobId, user));
this.shortLogsLink = StringHelper.Ujoin("logs", this.nodeId, this.containerId, jobId
, user);
}
}
protected override void Render(HtmlBlock.Block html)
{
string rmweb = $(AMParams.RmWeb);
Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> nav = html.Div("#nav").H3
("Cluster").Ul().Li().A(Url(rmweb, "cluster", "cluster"), "About").().Li().A(Url
(rmweb, "cluster", "apps"), "Applications").().Li().A(Url(rmweb, "cluster", "scheduler"
), "Scheduler").().().H3("Application").Ul().Li().A(Url("app/info"), "About").()
.Li().A(Url("app"), "Jobs").().();
if (app.GetJob() != null)
{
string jobid = MRApps.ToString(app.GetJob().GetID());
IList <AMInfo> amInfos = app.GetJob().GetAMInfos();
AMInfo thisAmInfo = amInfos[amInfos.Count - 1];
string nodeHttpAddress = thisAmInfo.GetNodeManagerHost() + ":" + thisAmInfo.GetNodeManagerHttpPort
();
nav.H3("Job").Ul().Li().A(Url("job", jobid), "Overview").().Li().A(Url("jobcounters"
, jobid), "Counters").().Li().A(Url("conf", jobid), "Configuration").().Li().A(Url
("tasks", jobid, "m"), "Map tasks").().Li().A(Url("tasks", jobid, "r"), "Reduce tasks"
).().Li().A(".logslink", Url(MRWebAppUtil.GetYARNWebappScheme(), nodeHttpAddress
, "node", "containerlogs", thisAmInfo.GetContainerId().ToString(), app.GetJob().
GetUserName()), "AM Logs").().();
if (app.GetTask() != null)
{
string taskid = MRApps.ToString(app.GetTask().GetID());
nav.H3("Task").Ul().Li().A(Url("task", taskid), "Task Overview").().Li().A(Url("taskcounters"
, taskid), "Counters").().();
}
}
nav.H3("Tools").Ul().Li().A("/conf", "Configuration").().Li().A("/logs", "Local logs"
).().Li().A("/stacks", "Server stacks").().Li().A("/jmx?qry=Hadoop:*", "Server metrics"
).().().();
}
public AMAttemptInfo(AMInfo amInfo, string jobId, string user)
{
this.nodeHttpAddress = string.Empty;
this.nodeId = string.Empty;
string nmHost = amInfo.GetNodeManagerHost();
int nmHttpPort = amInfo.GetNodeManagerHttpPort();
int nmPort = amInfo.GetNodeManagerPort();
if (nmHost != null)
{
this.nodeHttpAddress = nmHost + ":" + nmHttpPort;
NodeId nodeId = NodeId.NewInstance(nmHost, nmPort);
this.nodeId = nodeId.ToString();
}
this.id = amInfo.GetAppAttemptId().GetAttemptId();
this.startTime = amInfo.GetStartTime();
this.containerId = string.Empty;
this.logsLink = string.Empty;
ContainerId containerId = amInfo.GetContainerId();
if (containerId != null)
{
this.containerId = containerId.ToString();
this.logsLink = StringHelper.Join(MRWebAppUtil.GetYARNWebappScheme() + nodeHttpAddress
, StringHelper.Ujoin("node", "containerlogs", this.containerId, user));
}
}
public static AMInfo NewAMInfo(ApplicationAttemptId appAttemptId, long startTime,
ContainerId containerId, string nmHost, int nmPort, int nmHttpPort)
{
AMInfo amInfo = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AMInfo>();
amInfo.SetAppAttemptId(appAttemptId);
amInfo.SetStartTime(startTime);
amInfo.SetContainerId(containerId);
amInfo.SetNodeManagerHost(nmHost);
amInfo.SetNodeManagerPort(nmPort);
amInfo.SetNodeManagerHttpPort(nmHttpPort);
return(amInfo);
}
public virtual IList <AMInfo> GetAMInfos()
{
IList <AMInfo> amInfos = new List <AMInfo>();
foreach (JobHistoryParser.AMInfo jhAmInfo in jobInfo.GetAMInfos())
{
AMInfo amInfo = MRBuilderUtils.NewAMInfo(jhAmInfo.GetAppAttemptId(), jhAmInfo.GetStartTime
(), jhAmInfo.GetContainerId(), jhAmInfo.GetNodeManagerHost(), jhAmInfo.GetNodeManagerPort
(), jhAmInfo.GetNodeManagerHttpPort());
amInfos.AddItem(amInfo);
}
return(amInfos);
}
/// <exception cref="System.IO.IOException"/>
public virtual LogParams GetLogFilePath(JobID oldJobID, TaskAttemptID oldTaskAttemptID
)
{
JobId jobId = TypeConverter.ToYarn(oldJobID);
GetJobReportRequest request = recordFactory.NewRecordInstance <GetJobReportRequest
>();
request.SetJobId(jobId);
JobReport report = ((GetJobReportResponse)Invoke("getJobReport", typeof(GetJobReportRequest
), request)).GetJobReport();
if (EnumSet.Of(JobState.Succeeded, JobState.Failed, JobState.Killed, JobState.Error
).Contains(report.GetJobState()))
{
if (oldTaskAttemptID != null)
{
GetTaskAttemptReportRequest taRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest
>();
taRequest.SetTaskAttemptId(TypeConverter.ToYarn(oldTaskAttemptID));
TaskAttemptReport taReport = ((GetTaskAttemptReportResponse)Invoke("getTaskAttemptReport"
, typeof(GetTaskAttemptReportRequest), taRequest)).GetTaskAttemptReport();
if (taReport.GetContainerId() == null || taReport.GetNodeManagerHost() == null)
{
throw new IOException("Unable to get log information for task: " + oldTaskAttemptID
);
}
return(new LogParams(taReport.GetContainerId().ToString(), taReport.GetContainerId
().GetApplicationAttemptId().GetApplicationId().ToString(), NodeId.NewInstance(taReport
.GetNodeManagerHost(), taReport.GetNodeManagerPort()).ToString(), report.GetUser
()));
}
else
{
if (report.GetAMInfos() == null || report.GetAMInfos().Count == 0)
{
throw new IOException("Unable to get log information for job: " + oldJobID);
}
AMInfo amInfo = report.GetAMInfos()[report.GetAMInfos().Count - 1];
return(new LogParams(amInfo.GetContainerId().ToString(), amInfo.GetAppAttemptId()
.GetApplicationId().ToString(), NodeId.NewInstance(amInfo.GetNodeManagerHost(),
amInfo.GetNodeManagerPort()).ToString(), report.GetUser()));
}
}
else
{
throw new IOException("Cannot get log path for a in-progress job");
}
}
public virtual void TestAMInfosWithoutRecoveryEnabled()
{
int runCount = 0;
MRApp app = new TestRecovery.MRAppWithHistory(1, 0, false, this.GetType().FullName
, true, ++runCount);
Configuration conf = new Configuration();
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
long am1StartTime = app.GetAllAMInfos()[0].GetStartTime();
NUnit.Framework.Assert.AreEqual("No of tasks not correct", 1, job.GetTasks().Count
);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task mapTask = it.Next();
app.WaitForState(mapTask, TaskState.Running);
TaskAttempt taskAttempt = mapTask.GetAttempts().Values.GetEnumerator().Next();
app.WaitForState(taskAttempt, TaskAttemptState.Running);
// stop the app
app.Stop();
// rerun
app = new TestRecovery.MRAppWithHistory(1, 0, false, this.GetType().FullName, false
, ++runCount);
conf = new Configuration();
// in rerun the AMInfo will be recovered from previous run even if recovery
// is not enabled.
conf.SetBoolean(MRJobConfig.MrAmJobRecoveryEnable, false);
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("No of tasks not correct", 1, job.GetTasks().Count
);
it = job.GetTasks().Values.GetEnumerator();
mapTask = it.Next();
// There should be two AMInfos
IList <AMInfo> amInfos = app.GetAllAMInfos();
NUnit.Framework.Assert.AreEqual(2, amInfos.Count);
AMInfo amInfoOne = amInfos[0];
NUnit.Framework.Assert.AreEqual(am1StartTime, amInfoOne.GetStartTime());
app.Stop();
}
private void VerifyJobReport(JobReport jobReport, JobId jobId)
{
IList <AMInfo> amInfos = jobReport.GetAMInfos();
NUnit.Framework.Assert.AreEqual(1, amInfos.Count);
AMInfo amInfo = amInfos[0];
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(jobId.GetAppId
(), 1);
ContainerId amContainerId = ContainerId.NewContainerId(appAttemptId, 1);
NUnit.Framework.Assert.AreEqual(appAttemptId, amInfo.GetAppAttemptId());
NUnit.Framework.Assert.AreEqual(amContainerId, amInfo.GetContainerId());
NUnit.Framework.Assert.IsTrue(jobReport.GetSubmitTime() > 0);
NUnit.Framework.Assert.IsTrue(jobReport.GetStartTime() > 0 && jobReport.GetStartTime
() >= jobReport.GetSubmitTime());
NUnit.Framework.Assert.IsTrue(jobReport.GetFinishTime() > 0 && jobReport.GetFinishTime
() >= jobReport.GetStartTime());
}
// pass
private void VerifyJobReport(JobReport jr)
{
NUnit.Framework.Assert.IsNotNull("JobReport is null", jr);
IList <AMInfo> amInfos = jr.GetAMInfos();
NUnit.Framework.Assert.AreEqual(1, amInfos.Count);
NUnit.Framework.Assert.AreEqual(JobState.Running, jr.GetJobState());
AMInfo amInfo = amInfos[0];
NUnit.Framework.Assert.AreEqual(MRApp.NmHost, amInfo.GetNodeManagerHost());
NUnit.Framework.Assert.AreEqual(MRApp.NmPort, amInfo.GetNodeManagerPort());
NUnit.Framework.Assert.AreEqual(MRApp.NmHttpPort, amInfo.GetNodeManagerHttpPort()
);
NUnit.Framework.Assert.AreEqual(1, amInfo.GetAppAttemptId().GetAttemptId());
NUnit.Framework.Assert.AreEqual(1, amInfo.GetContainerId().GetApplicationAttemptId
().GetAttemptId());
NUnit.Framework.Assert.IsTrue(amInfo.GetStartTime() > 0);
NUnit.Framework.Assert.AreEqual(false, jr.IsUber());
}
private MRProtos.AMInfoProto ConvertToProtoFormat(AMInfo t)
{
return(((AMInfoPBImpl)t).GetProto());
}