private void VerifyEnemyAppReport(ApplicationReport appReport)
{
NUnit.Framework.Assert.AreEqual("Enemy should not see app host!", Unavailable, appReport
.GetHost());
NUnit.Framework.Assert.AreEqual("Enemy should not see app rpc port!", -1, appReport
.GetRpcPort());
NUnit.Framework.Assert.AreEqual("Enemy should not see app client token!", null, appReport
.GetClientToAMToken());
NUnit.Framework.Assert.AreEqual("Enemy should not see app diagnostics!", Unavailable
, appReport.GetDiagnostics());
NUnit.Framework.Assert.AreEqual("Enemy should not see app tracking url!", Unavailable
, appReport.GetTrackingUrl());
NUnit.Framework.Assert.AreEqual("Enemy should not see app original tracking url!"
, Unavailable, appReport.GetOriginalTrackingUrl());
ApplicationResourceUsageReport usageReport = appReport.GetApplicationResourceUsageReport
();
NUnit.Framework.Assert.AreEqual("Enemy should not see app used containers", -1, usageReport
.GetNumUsedContainers());
NUnit.Framework.Assert.AreEqual("Enemy should not see app reserved containers", -
1, usageReport.GetNumReservedContainers());
NUnit.Framework.Assert.AreEqual("Enemy should not see app used resources", -1, usageReport
.GetUsedResources().GetMemory());
NUnit.Framework.Assert.AreEqual("Enemy should not see app reserved resources", -1
, usageReport.GetReservedResources().GetMemory());
NUnit.Framework.Assert.AreEqual("Enemy should not see app needed resources", -1,
usageReport.GetNeededResources().GetMemory());
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override JobStatus SubmitJob(JobID jobId, string jobSubmitDir, Credentials
ts)
{
AddHistoryToken(ts);
// Construct necessary information to start the MR AM
ApplicationSubmissionContext appContext = CreateApplicationSubmissionContext(conf
, jobSubmitDir, ts);
// Submit to ResourceManager
try
{
ApplicationId applicationId = resMgrDelegate.SubmitApplication(appContext);
ApplicationReport appMaster = resMgrDelegate.GetApplicationReport(applicationId);
string diagnostics = (appMaster == null ? "application report is null" : appMaster
.GetDiagnostics());
if (appMaster == null || appMaster.GetYarnApplicationState() == YarnApplicationState
.Failed || appMaster.GetYarnApplicationState() == YarnApplicationState.Killed)
{
throw new IOException("Failed to run job : " + diagnostics);
}
return(clientCache.GetClient(jobId).GetJobStatus(jobId));
}
catch (YarnException e)
{
throw new IOException(e);
}
}
public static ApplicationReport NewApplicationReport(ApplicationId applicationId,
ApplicationAttemptId applicationAttemptId, string user, string queue, string name
, string host, int rpcPort, Org.Apache.Hadoop.Yarn.Api.Records.Token clientToAMToken
, YarnApplicationState state, string diagnostics, string url, long startTime, long
finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport
appResources, string origTrackingUrl, float progress, string appType, Org.Apache.Hadoop.Yarn.Api.Records.Token
amRmToken, ICollection <string> tags)
{
ApplicationReport report = recordFactory.NewRecordInstance <ApplicationReport>();
report.SetApplicationId(applicationId);
report.SetCurrentApplicationAttemptId(applicationAttemptId);
report.SetUser(user);
report.SetQueue(queue);
report.SetName(name);
report.SetHost(host);
report.SetRpcPort(rpcPort);
report.SetClientToAMToken(clientToAMToken);
report.SetYarnApplicationState(state);
report.SetDiagnostics(diagnostics);
report.SetTrackingUrl(url);
report.SetStartTime(startTime);
report.SetFinishTime(finishTime);
report.SetFinalApplicationStatus(finalStatus);
report.SetApplicationResourceUsageReport(appResources);
report.SetOriginalTrackingUrl(origTrackingUrl);
report.SetProgress(progress);
report.SetApplicationType(appType);
report.SetAMRMToken(amRmToken);
report.SetApplicationTags(tags);
return(report);
}
/// <exception cref="System.Exception"/>
public virtual void TestGetApplicationReportOnHA()
{
ApplicationReport report = client.GetApplicationReport(cluster.CreateFakeAppId());
NUnit.Framework.Assert.IsTrue(report != null);
NUnit.Framework.Assert.AreEqual(cluster.CreateFakeAppReport(), report);
}
public static JobStatus FromYarn(ApplicationReport application, string jobFile)
{
string trackingUrl = application.GetTrackingUrl();
trackingUrl = trackingUrl == null ? string.Empty : trackingUrl;
JobStatus jobStatus = new JobStatus(TypeConverter.FromYarn(application.GetApplicationId
()), 0.0f, 0.0f, 0.0f, 0.0f, TypeConverter.FromYarn(application.GetYarnApplicationState
(), application.GetFinalApplicationStatus()), JobPriority.Normal, application.GetUser
(), application.GetName(), application.GetQueue(), jobFile, trackingUrl, false);
jobStatus.SetSchedulingInfo(trackingUrl);
// Set AM tracking url
jobStatus.SetStartTime(application.GetStartTime());
jobStatus.SetFinishTime(application.GetFinishTime());
jobStatus.SetFailureInfo(application.GetDiagnostics());
ApplicationResourceUsageReport resourceUsageReport = application.GetApplicationResourceUsageReport
();
if (resourceUsageReport != null)
{
jobStatus.SetNeededMem(resourceUsageReport.GetNeededResources().GetMemory());
jobStatus.SetNumReservedSlots(resourceUsageReport.GetNumReservedContainers());
jobStatus.SetNumUsedSlots(resourceUsageReport.GetNumUsedContainers());
jobStatus.SetReservedMem(resourceUsageReport.GetReservedResources().GetMemory());
jobStatus.SetUsedMem(resourceUsageReport.GetUsedResources().GetMemory());
}
return(jobStatus);
}
public static ClientRMService MockClientRMService(RMContext rmContext)
{
ClientRMService clientRMService = Org.Mockito.Mockito.Mock <ClientRMService>();
IList <ApplicationReport> appReports = new AList <ApplicationReport>();
foreach (RMApp app in rmContext.GetRMApps().Values)
{
ApplicationReport appReport = ApplicationReport.NewInstance(app.GetApplicationId(
), (ApplicationAttemptId)null, app.GetUser(), app.GetQueue(), app.GetName(), (string
)null, 0, (Token)null, app.CreateApplicationState(), app.GetDiagnostics().ToString
(), (string)null, app.GetStartTime(), app.GetFinishTime(), app.GetFinalApplicationStatus
(), (ApplicationResourceUsageReport)null, app.GetTrackingUrl(), app.GetProgress(
), app.GetApplicationType(), (Token)null);
appReports.AddItem(appReport);
}
GetApplicationsResponse response = Org.Mockito.Mockito.Mock <GetApplicationsResponse
>();
Org.Mockito.Mockito.When(response.GetApplicationList()).ThenReturn(appReports);
try
{
Org.Mockito.Mockito.When(clientRMService.GetApplications(Matchers.Any <GetApplicationsRequest
>())).ThenReturn(response);
}
catch (YarnException)
{
NUnit.Framework.Assert.Fail("Exception is not expteced.");
}
return(clientRMService);
}
/// <summary>Kills the application with the application id as appId</summary>
/// <param name="applicationId"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private void KillApplication(string applicationId)
{
ApplicationId appId = ConverterUtils.ToApplicationId(applicationId);
ApplicationReport appReport = null;
try
{
appReport = client.GetApplicationReport(appId);
}
catch (ApplicationNotFoundException e)
{
sysout.WriteLine("Application with id '" + applicationId + "' doesn't exist in RM."
);
throw;
}
if (appReport.GetYarnApplicationState() == YarnApplicationState.Finished || appReport
.GetYarnApplicationState() == YarnApplicationState.Killed || appReport.GetYarnApplicationState
() == YarnApplicationState.Failed)
{
sysout.WriteLine("Application " + applicationId + " has already finished ");
}
else
{
sysout.WriteLine("Killing application " + applicationId);
client.KillApplication(appId);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private int VerifyApplicationState(ApplicationId appId)
{
YarnClient yarnClient = CreateYarnClient();
try
{
ApplicationReport appReport = yarnClient.GetApplicationReport(appId);
switch (appReport.GetYarnApplicationState())
{
case YarnApplicationState.New:
case YarnApplicationState.NewSaving:
case YarnApplicationState.Submitted:
{
return(-1);
}
case YarnApplicationState.Accepted:
case YarnApplicationState.Running:
case YarnApplicationState.Failed:
case YarnApplicationState.Finished:
case YarnApplicationState.Killed:
default:
{
break;
}
}
}
finally
{
yarnClient.Close();
}
return(0);
}
public virtual void TestFromYarn()
{
int appStartTime = 612354;
int appFinishTime = 612355;
YarnApplicationState state = YarnApplicationState.Running;
ApplicationId applicationId = ApplicationId.NewInstance(0, 0);
ApplicationReport applicationReport = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ApplicationReport>();
applicationReport.SetApplicationId(applicationId);
applicationReport.SetYarnApplicationState(state);
applicationReport.SetStartTime(appStartTime);
applicationReport.SetFinishTime(appFinishTime);
applicationReport.SetUser("TestTypeConverter-user");
ApplicationResourceUsageReport appUsageRpt = Org.Apache.Hadoop.Yarn.Util.Records.
NewRecord <ApplicationResourceUsageReport>();
Resource r = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Resource>();
r.SetMemory(2048);
appUsageRpt.SetNeededResources(r);
appUsageRpt.SetNumReservedContainers(1);
appUsageRpt.SetNumUsedContainers(3);
appUsageRpt.SetReservedResources(r);
appUsageRpt.SetUsedResources(r);
applicationReport.SetApplicationResourceUsageReport(appUsageRpt);
JobStatus jobStatus = TypeConverter.FromYarn(applicationReport, "dummy-jobfile");
NUnit.Framework.Assert.AreEqual(appStartTime, jobStatus.GetStartTime());
NUnit.Framework.Assert.AreEqual(appFinishTime, jobStatus.GetFinishTime());
NUnit.Framework.Assert.AreEqual(state.ToString(), jobStatus.GetState().ToString()
);
}
/// <exception cref="System.IO.IOException"/>
public virtual ContainerReport GetAMContainer(ApplicationAttemptId appAttemptId)
{
ApplicationReport app = GetApplication(appAttemptId.GetApplicationId());
return(ConvertToContainerReport(historyStore.GetAMContainer(appAttemptId), app ==
null ? null : app.GetUser()));
}
public AppInfo(ApplicationReport app)
{
// JAXB needs this
appId = app.GetApplicationId().ToString();
if (app.GetCurrentApplicationAttemptId() != null)
{
currentAppAttemptId = app.GetCurrentApplicationAttemptId().ToString();
}
user = app.GetUser();
queue = app.GetQueue();
name = app.GetName();
type = app.GetApplicationType();
host = app.GetHost();
rpcPort = app.GetRpcPort();
appState = app.GetYarnApplicationState();
diagnosticsInfo = app.GetDiagnostics();
trackingUrl = app.GetTrackingUrl();
originalTrackingUrl = app.GetOriginalTrackingUrl();
submittedTime = app.GetStartTime();
startedTime = app.GetStartTime();
finishedTime = app.GetFinishTime();
elapsedTime = Times.Elapsed(startedTime, finishedTime);
finalAppStatus = app.GetFinalApplicationStatus();
progress = app.GetProgress() * 100;
// in percent
if (app.GetApplicationTags() != null && !app.GetApplicationTags().IsEmpty())
{
this.applicationTags = StringHelper.CsvJoiner.Join(app.GetApplicationTags());
}
}
/// <exception cref="System.Exception"/>
public virtual void TestInvalidatedAMHostPortOnAMRestart()
{
MockRM rm1 = new MockRM(conf);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());
nm1.RegisterNode();
// a failed app
RMApp app2 = rm1.SubmitApp(200);
MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm1);
nm1.NodeHeartbeat(am2.GetApplicationAttemptId(), 1, ContainerState.Complete);
am2.WaitForState(RMAppAttemptState.Failed);
rm1.WaitForState(app2.GetApplicationId(), RMAppState.Accepted);
// before new attempt is launched, the app report returns the invalid AM
// host and port.
GetApplicationReportRequest request1 = GetApplicationReportRequest.NewInstance(app2
.GetApplicationId());
ApplicationReport report1 = rm1.GetClientRMService().GetApplicationReport(request1
).GetApplicationReport();
NUnit.Framework.Assert.AreEqual("N/A", report1.GetHost());
NUnit.Framework.Assert.AreEqual(-1, report1.GetRpcPort());
}
public virtual AppInfo GetApp(HttpServletRequest req, HttpServletResponse res, string
appId)
{
UserGroupInformation callerUGI = GetUser(req);
ApplicationId id = ParseApplicationId(appId);
ApplicationReport app = null;
try
{
if (callerUGI == null)
{
GetApplicationReportRequest request = GetApplicationReportRequest.NewInstance(id);
app = appBaseProt.GetApplicationReport(request).GetApplicationReport();
}
else
{
app = callerUGI.DoAs(new _PrivilegedExceptionAction_228(this, id));
}
}
catch (Exception e)
{
RewrapAndThrowException(e);
}
if (app == null)
{
throw new NotFoundException("app with id: " + appId + " not found");
}
return(new AppInfo(app));
}
/// <exception cref="System.Exception"/>
public virtual void TestJobSubmissionFailure()
{
Org.Mockito.Mockito.When(resourceMgrDelegate.SubmitApplication(Matchers.Any <ApplicationSubmissionContext
>())).ThenReturn(appId);
ApplicationReport report = Org.Mockito.Mockito.Mock <ApplicationReport>();
Org.Mockito.Mockito.When(report.GetApplicationId()).ThenReturn(appId);
Org.Mockito.Mockito.When(report.GetDiagnostics()).ThenReturn(failString);
Org.Mockito.Mockito.When(report.GetYarnApplicationState()).ThenReturn(YarnApplicationState
.Failed);
Org.Mockito.Mockito.When(resourceMgrDelegate.GetApplicationReport(appId)).ThenReturn
(report);
Credentials credentials = new Credentials();
FilePath jobxml = new FilePath(testWorkDir, "job.xml");
OutputStream @out = new FileOutputStream(jobxml);
conf.WriteXml(@out);
@out.Close();
try
{
yarnRunner.SubmitJob(jobId, testWorkDir.GetAbsolutePath().ToString(), credentials
);
}
catch (IOException io)
{
Log.Info("Logging exception:", io);
NUnit.Framework.Assert.IsTrue(io.GetLocalizedMessage().Contains(failString));
}
}
public ApplicationReportExt(ApplicationReport appReport, IDictionary <ApplicationAccessType
, string> appViewACLs)
{
// retrieve all the fields
// retrieve user and ACLs info only
this.appReport = appReport;
this.appViewACLs = appViewACLs;
}
private ApplicationReport GetRunningApplicationReport(string host, int port)
{
ApplicationId appId = ApplicationId.NewInstance(1234, 5);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 0);
return(ApplicationReport.NewInstance(appId, attemptId, "user", "queue", "appname"
, host, port, null, YarnApplicationState.Running, "diagnostics", "url", 0, 0, FinalApplicationStatus
.Undefined, null, "N/A", 0.0f, YarnConfiguration.DefaultApplicationType, null));
}
public static GetApplicationReportResponse NewInstance(ApplicationReport ApplicationReport
)
{
GetApplicationReportResponse response = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<GetApplicationReportResponse>();
response.SetApplicationReport(ApplicationReport);
return(response);
}
public override void SetApplicationReport(ApplicationReport applicationMaster)
{
MaybeInitBuilder();
if (applicationMaster == null)
{
builder.ClearApplicationReport();
}
this.applicationReport = applicationMaster;
}
private ApplicationReport GetFinishedApplicationReport()
{
ApplicationId appId = ApplicationId.NewInstance(1234, 5);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 0);
return(ApplicationReport.NewInstance(appId, attemptId, "user", "queue", "appname"
, "host", 124, null, YarnApplicationState.Finished, "diagnostics", "url", 0, 0,
FinalApplicationStatus.Succeeded, null, "N/A", 0.0f, YarnConfiguration.DefaultApplicationType
, null));
}
private MRClientProtocol CheckAndGetHSProxy(ApplicationReport applicationReport,
JobState state)
{
if (null == historyServerProxy)
{
Log.Warn("Job History Server is not configured.");
return(GetNotRunningJob(applicationReport, state));
}
return(historyServerProxy);
}
/// <summary>Monitor the submitted application for completion.</summary>
/// <remarks>
/// Monitor the submitted application for completion.
/// Kill application if time expires.
/// </remarks>
/// <param name="appId">Application Id of application to be monitored</param>
/// <returns>true if application completed successfully</returns>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private bool MonitorApplication(ApplicationId appId)
{
while (true)
{
// Check app status every 1 second.
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
Log.Debug("Thread sleep in monitoring loop interrupted");
}
// Get application report for the appId we are interested in
ApplicationReport report = yarnClient.GetApplicationReport(appId);
Log.Info("Got application report from ASM for" + ", appId=" + appId.GetId() + ", clientToAMToken="
+ report.GetClientToAMToken() + ", appDiagnostics=" + report.GetDiagnostics() +
", appMasterHost=" + report.GetHost() + ", appQueue=" + report.GetQueue() + ", appMasterRpcPort="
+ report.GetRpcPort() + ", appStartTime=" + report.GetStartTime() + ", yarnAppState="
+ report.GetYarnApplicationState().ToString() + ", distributedFinalState=" + report
.GetFinalApplicationStatus().ToString() + ", appTrackingUrl=" + report.GetTrackingUrl
() + ", appUser="******"Application has completed successfully. Breaking monitoring loop");
return(true);
}
else
{
Log.Info("Application did finished unsuccessfully." + " YarnState=" + state.ToString
() + ", DSFinalStatus=" + dsStatus.ToString() + ". Breaking monitoring loop");
return(false);
}
}
else
{
if (YarnApplicationState.Killed == state || YarnApplicationState.Failed == state)
{
Log.Info("Application did not finish." + " YarnState=" + state.ToString() + ", DSFinalStatus="
+ dsStatus.ToString() + ". Breaking monitoring loop");
return(false);
}
}
if (Runtime.CurrentTimeMillis() > (clientStartTime + clientTimeout))
{
Log.Info("Reached client specified timeout for application. Killing application");
ForceKillApplication(appId);
return(false);
}
}
}
public FetchedAppReport(ApplicationReport appReport, AppReportFetcher.AppReportSource
appReportSource)
{
/*
* This class creates a bundle of the application report and the source from
* where the the report was fetched. This allows the WebAppProxyServlet
* to make decisions for the application report based on the source.
*/
this.appReport = appReport;
this.appReportSource = appReportSource;
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private YarnClient CreateMockYarnClient(YarnApplicationState appState)
{
YarnClient mockClient = Org.Mockito.Mockito.Mock <YarnClient>();
ApplicationReport mockAppReport = Org.Mockito.Mockito.Mock <ApplicationReport>();
Org.Mockito.Mockito.DoReturn(appState).When(mockAppReport).GetYarnApplicationState
();
Org.Mockito.Mockito.DoReturn(mockAppReport).When(mockClient).GetApplicationReport
(Matchers.Any <ApplicationId>());
return(mockClient);
}
private ApplicationReport GetUnknownApplicationReport()
{
ApplicationId unknownAppId = recordFactory.NewRecordInstance <ApplicationId>();
ApplicationAttemptId unknownAttemptId = recordFactory.NewRecordInstance <ApplicationAttemptId
>();
// Setting AppState to NEW and finalStatus to UNDEFINED as they are never
// used for a non running job
return(ApplicationReport.NewInstance(unknownAppId, unknownAttemptId, "N/A", "N/A"
, "N/A", "N/A", 0, null, YarnApplicationState.New, "N/A", "N/A", 0, 0, FinalApplicationStatus
.Undefined, null, "N/A", 0.0f, YarnConfiguration.DefaultApplicationType, null));
}
public virtual ApplicationReport CreateFakeAppReport()
{
ApplicationId appId = ApplicationId.NewInstance(1000l, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
// create a fake application report
ApplicationReport report = ApplicationReport.NewInstance(appId, attemptId, "fakeUser"
, "fakeQueue", "fakeApplicationName", "localhost", 0, null, YarnApplicationState
.Finished, "fake an application report", string.Empty, 1000l, 1200l, FinalApplicationStatus
.Failed, null, string.Empty, 50f, "fakeApplicationType", null);
return(report);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public override GetApplicationReportResponse GetApplicationReport(GetApplicationReportRequest
request)
{
this._enclosing.ResetStartFailoverFlag(true);
// make sure failover has been triggered
NUnit.Framework.Assert.IsTrue(this._enclosing.WaittingForFailOver());
// create a fake application report
ApplicationReport report = this._enclosing.CreateFakeAppReport();
GetApplicationReportResponse response = GetApplicationReportResponse.NewInstance(
report);
return(response);
}
private static GetApplicationReportResponse CreateApplicationReportWithFinishedApplication
()
{
ApplicationReport report = Org.Mockito.Mockito.Mock <ApplicationReport>();
Org.Mockito.Mockito.When(report.GetYarnApplicationState()).ThenReturn(YarnApplicationState
.Finished);
GetApplicationReportResponse response = Org.Mockito.Mockito.Mock <GetApplicationReportResponse
>();
Org.Mockito.Mockito.When(response.GetApplicationReport()).ThenReturn(report);
return(response);
}
public override ApplicationReport CreateAndGetApplicationReport(string clientUserName
, bool allowAccess)
{
ApplicationResourceUsageReport usageReport = ApplicationResourceUsageReport.NewInstance
(0, 0, null, null, null, 0, 0);
ApplicationReport report = ApplicationReport.NewInstance(this.GetApplicationId(),
appAttemptId, this.GetUser(), this.GetQueue(), this.GetName(), null, 0, null, null
, this.GetDiagnostics().ToString(), this.GetTrackingUrl(), this.GetStartTime(),
this.GetFinishTime(), this.GetFinalApplicationStatus(), usageReport, null, this.
GetProgress(), type, null);
return(report);
}
protected internal static ApplicationReport CreateApplicationReport(int appIdInt,
int appAttemptIdInt, long timestamp)
{
ApplicationId appId = ApplicationId.NewInstance(timestamp, appIdInt);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, appAttemptIdInt
);
ApplicationReport appReport = ApplicationReport.NewInstance(appId, appAttemptId,
"user", "queue", "appname", "host", 124, null, YarnApplicationState.Finished, "diagnostics"
, "url", 0, 0, FinalApplicationStatus.Succeeded, null, "N/A", 0.53789f, YarnConfiguration
.DefaultApplicationType, null);
return(appReport);
}
public override ApplicationReport GetApplicationReport()
{
YarnServiceProtos.GetApplicationReportResponseProtoOrBuilder p = viaProto ? proto
: builder;
if (this.applicationReport != null)
{
return(this.applicationReport);
}
if (!p.HasApplicationReport())
{
return(null);
}
this.applicationReport = ConvertFromProtoFormat(p.GetApplicationReport());
return(this.applicationReport);
}