public static JobStatus FromYarn(ApplicationReport application, string jobFile)
{
string trackingUrl = application.GetTrackingUrl();
trackingUrl = trackingUrl == null ? string.Empty : trackingUrl;
JobStatus jobStatus = new JobStatus(TypeConverter.FromYarn(application.GetApplicationId
()), 0.0f, 0.0f, 0.0f, 0.0f, TypeConverter.FromYarn(application.GetYarnApplicationState
(), application.GetFinalApplicationStatus()), JobPriority.Normal, application.GetUser
(), application.GetName(), application.GetQueue(), jobFile, trackingUrl, false);
jobStatus.SetSchedulingInfo(trackingUrl);
// Set AM tracking url
jobStatus.SetStartTime(application.GetStartTime());
jobStatus.SetFinishTime(application.GetFinishTime());
jobStatus.SetFailureInfo(application.GetDiagnostics());
ApplicationResourceUsageReport resourceUsageReport = application.GetApplicationResourceUsageReport
();
if (resourceUsageReport != null)
{
jobStatus.SetNeededMem(resourceUsageReport.GetNeededResources().GetMemory());
jobStatus.SetNumReservedSlots(resourceUsageReport.GetNumReservedContainers());
jobStatus.SetNumUsedSlots(resourceUsageReport.GetNumUsedContainers());
jobStatus.SetReservedMem(resourceUsageReport.GetReservedResources().GetMemory());
jobStatus.SetUsedMem(resourceUsageReport.GetUsedResources().GetMemory());
}
return(jobStatus);
}
private void VerifyEnemyAppReport(ApplicationReport appReport)
{
NUnit.Framework.Assert.AreEqual("Enemy should not see app host!", Unavailable, appReport
.GetHost());
NUnit.Framework.Assert.AreEqual("Enemy should not see app rpc port!", -1, appReport
.GetRpcPort());
NUnit.Framework.Assert.AreEqual("Enemy should not see app client token!", null, appReport
.GetClientToAMToken());
NUnit.Framework.Assert.AreEqual("Enemy should not see app diagnostics!", Unavailable
, appReport.GetDiagnostics());
NUnit.Framework.Assert.AreEqual("Enemy should not see app tracking url!", Unavailable
, appReport.GetTrackingUrl());
NUnit.Framework.Assert.AreEqual("Enemy should not see app original tracking url!"
, Unavailable, appReport.GetOriginalTrackingUrl());
ApplicationResourceUsageReport usageReport = appReport.GetApplicationResourceUsageReport
();
NUnit.Framework.Assert.AreEqual("Enemy should not see app used containers", -1, usageReport
.GetNumUsedContainers());
NUnit.Framework.Assert.AreEqual("Enemy should not see app reserved containers", -
1, usageReport.GetNumReservedContainers());
NUnit.Framework.Assert.AreEqual("Enemy should not see app used resources", -1, usageReport
.GetUsedResources().GetMemory());
NUnit.Framework.Assert.AreEqual("Enemy should not see app reserved resources", -1
, usageReport.GetReservedResources().GetMemory());
NUnit.Framework.Assert.AreEqual("Enemy should not see app needed resources", -1,
usageReport.GetNeededResources().GetMemory());
}
public virtual void TestApplicationReport()
{
ApplicationId appId = null;
appId = ApplicationId.NewInstance(0, 1);
GetApplicationReportRequest request = GetApplicationReportRequest.NewInstance(appId
);
GetApplicationReportResponse response = clientService.GetApplicationReport(request
);
ApplicationReport appReport = response.GetApplicationReport();
NUnit.Framework.Assert.IsNotNull(appReport);
NUnit.Framework.Assert.AreEqual(123, appReport.GetApplicationResourceUsageReport(
).GetMemorySeconds());
NUnit.Framework.Assert.AreEqual(345, appReport.GetApplicationResourceUsageReport(
).GetVcoreSeconds());
NUnit.Framework.Assert.AreEqual("application_0_0001", appReport.GetApplicationId(
).ToString());
NUnit.Framework.Assert.AreEqual("test app type", appReport.GetApplicationType().ToString
());
NUnit.Framework.Assert.AreEqual("test queue", appReport.GetQueue().ToString());
}
private ApplicationReport GetApplicationReport(YarnApplicationState yarnApplicationState
, FinalApplicationStatus finalApplicationStatus)
{
ApplicationReport appReport = Org.Mockito.Mockito.Mock <ApplicationReport>();
ApplicationResourceUsageReport appResources = Org.Mockito.Mockito.Mock <ApplicationResourceUsageReport
>();
Org.Mockito.Mockito.When(appReport.GetApplicationId()).ThenReturn(ApplicationId.NewInstance
(0, 0));
Org.Mockito.Mockito.When(appResources.GetNeededResources()).ThenReturn(Org.Apache.Hadoop.Yarn.Util.Records
.NewRecord <Resource>());
Org.Mockito.Mockito.When(appResources.GetReservedResources()).ThenReturn(Org.Apache.Hadoop.Yarn.Util.Records
.NewRecord <Resource>());
Org.Mockito.Mockito.When(appResources.GetUsedResources()).ThenReturn(Org.Apache.Hadoop.Yarn.Util.Records
.NewRecord <Resource>());
Org.Mockito.Mockito.When(appReport.GetApplicationResourceUsageReport()).ThenReturn
(appResources);
Org.Mockito.Mockito.When(appReport.GetYarnApplicationState()).ThenReturn(yarnApplicationState
);
Org.Mockito.Mockito.When(appReport.GetFinalApplicationStatus()).ThenReturn(finalApplicationStatus
);
return(appReport);
}
public virtual void TestFromYarnApplicationReport()
{
ApplicationId mockAppId = Org.Mockito.Mockito.Mock <ApplicationId>();
Org.Mockito.Mockito.When(mockAppId.GetClusterTimestamp()).ThenReturn(12345L);
Org.Mockito.Mockito.When(mockAppId.GetId()).ThenReturn(6789);
ApplicationReport mockReport = Org.Mockito.Mockito.Mock <ApplicationReport>();
Org.Mockito.Mockito.When(mockReport.GetTrackingUrl()).ThenReturn("dummy-tracking-url"
);
Org.Mockito.Mockito.When(mockReport.GetApplicationId()).ThenReturn(mockAppId);
Org.Mockito.Mockito.When(mockReport.GetYarnApplicationState()).ThenReturn(YarnApplicationState
.Killed);
Org.Mockito.Mockito.When(mockReport.GetUser()).ThenReturn("dummy-user");
Org.Mockito.Mockito.When(mockReport.GetQueue()).ThenReturn("dummy-queue");
string jobFile = "dummy-path/job.xml";
try
{
JobStatus status = TypeConverter.FromYarn(mockReport, jobFile);
}
catch (ArgumentNullException)
{
NUnit.Framework.Assert.Fail("Type converstion from YARN fails for jobs without "
+ "ApplicationUsageReport");
}
ApplicationResourceUsageReport appUsageRpt = Org.Apache.Hadoop.Yarn.Util.Records.
NewRecord <ApplicationResourceUsageReport>();
Resource r = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Resource>();
r.SetMemory(2048);
appUsageRpt.SetNeededResources(r);
appUsageRpt.SetNumReservedContainers(1);
appUsageRpt.SetNumUsedContainers(3);
appUsageRpt.SetReservedResources(r);
appUsageRpt.SetUsedResources(r);
Org.Mockito.Mockito.When(mockReport.GetApplicationResourceUsageReport()).ThenReturn
(appUsageRpt);
JobStatus status_1 = TypeConverter.FromYarn(mockReport, jobFile);
NUnit.Framework.Assert.IsNotNull("fromYarn returned null status", status_1);
NUnit.Framework.Assert.AreEqual("jobFile set incorrectly", "dummy-path/job.xml",
status_1.GetJobFile());
NUnit.Framework.Assert.AreEqual("queue set incorrectly", "dummy-queue", status_1.
GetQueue());
NUnit.Framework.Assert.AreEqual("trackingUrl set incorrectly", "dummy-tracking-url"
, status_1.GetTrackingUrl());
NUnit.Framework.Assert.AreEqual("user set incorrectly", "dummy-user", status_1.GetUsername
());
NUnit.Framework.Assert.AreEqual("schedulingInfo set incorrectly", "dummy-tracking-url"
, status_1.GetSchedulingInfo());
NUnit.Framework.Assert.AreEqual("jobId set incorrectly", 6789, status_1.GetJobID(
).GetId());
NUnit.Framework.Assert.AreEqual("state set incorrectly", JobStatus.State.Killed,
status_1.GetState());
NUnit.Framework.Assert.AreEqual("needed mem info set incorrectly", 2048, status_1
.GetNeededMem());
NUnit.Framework.Assert.AreEqual("num rsvd slots info set incorrectly", 1, status_1
.GetNumReservedSlots());
NUnit.Framework.Assert.AreEqual("num used slots info set incorrectly", 3, status_1
.GetNumUsedSlots());
NUnit.Framework.Assert.AreEqual("rsvd mem info set incorrectly", 2048, status_1.GetReservedMem
());
NUnit.Framework.Assert.AreEqual("used mem info set incorrectly", 2048, status_1.GetUsedMem
());
}
/// <summary>Prints the application report for an application id.</summary>
/// <param name="applicationId"/>
/// <returns>exitCode</returns>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private int PrintApplicationReport(string applicationId)
{
ApplicationReport appReport = null;
try
{
appReport = client.GetApplicationReport(ConverterUtils.ToApplicationId(applicationId
));
}
catch (ApplicationNotFoundException)
{
sysout.WriteLine("Application with id '" + applicationId + "' doesn't exist in RM or Timeline Server."
);
return(-1);
}
// Use PrintWriter.println, which uses correct platform line ending.
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter appReportStr = new PrintWriter(new OutputStreamWriter(baos, Sharpen.Extensions.GetEncoding
("UTF-8")));
if (appReport != null)
{
appReportStr.WriteLine("Application Report : ");
appReportStr.Write("\tApplication-Id : ");
appReportStr.WriteLine(appReport.GetApplicationId());
appReportStr.Write("\tApplication-Name : ");
appReportStr.WriteLine(appReport.GetName());
appReportStr.Write("\tApplication-Type : ");
appReportStr.WriteLine(appReport.GetApplicationType());
appReportStr.Write("\tUser : "******"\tQueue : ");
appReportStr.WriteLine(appReport.GetQueue());
appReportStr.Write("\tStart-Time : ");
appReportStr.WriteLine(appReport.GetStartTime());
appReportStr.Write("\tFinish-Time : ");
appReportStr.WriteLine(appReport.GetFinishTime());
appReportStr.Write("\tProgress : ");
DecimalFormat formatter = new DecimalFormat("###.##%");
string progress = formatter.Format(appReport.GetProgress());
appReportStr.WriteLine(progress);
appReportStr.Write("\tState : ");
appReportStr.WriteLine(appReport.GetYarnApplicationState());
appReportStr.Write("\tFinal-State : ");
appReportStr.WriteLine(appReport.GetFinalApplicationStatus());
appReportStr.Write("\tTracking-URL : ");
appReportStr.WriteLine(appReport.GetOriginalTrackingUrl());
appReportStr.Write("\tRPC Port : ");
appReportStr.WriteLine(appReport.GetRpcPort());
appReportStr.Write("\tAM Host : ");
appReportStr.WriteLine(appReport.GetHost());
appReportStr.Write("\tAggregate Resource Allocation : ");
ApplicationResourceUsageReport usageReport = appReport.GetApplicationResourceUsageReport
();
if (usageReport != null)
{
//completed app report in the timeline server doesn't have usage report
appReportStr.Write(usageReport.GetMemorySeconds() + " MB-seconds, ");
appReportStr.WriteLine(usageReport.GetVcoreSeconds() + " vcore-seconds");
}
else
{
appReportStr.WriteLine("N/A");
}
appReportStr.Write("\tDiagnostics : ");
appReportStr.Write(appReport.GetDiagnostics());
}
else
{
appReportStr.Write("Application with id '" + applicationId + "' doesn't exist in RM."
);
appReportStr.Close();
sysout.WriteLine(baos.ToString("UTF-8"));
return(-1);
}
appReportStr.Close();
sysout.WriteLine(baos.ToString("UTF-8"));
return(0);
}