private static RMApp CreateRMApp(ApplicationId appId)
{
RMApp app = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(appId);
Org.Mockito.Mockito.When(app.GetName()).ThenReturn("test app");
Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("test app type");
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("test user");
Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("test queue");
Org.Mockito.Mockito.When(app.GetSubmitTime()).ThenReturn(int.MaxValue + 1L);
Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(int.MaxValue + 2L);
Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(int.MaxValue + 3L);
Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder("test diagnostics info"
));
RMAppAttempt appAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(appAttempt.GetAppAttemptId()).ThenReturn(ApplicationAttemptId
.NewInstance(appId, 1));
Org.Mockito.Mockito.When(app.GetCurrentAppAttempt()).ThenReturn(appAttempt);
Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
.Undefined);
Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(new RMAppMetrics(null,
0, 0, int.MaxValue, long.MaxValue));
return(app);
}
public virtual void TestEscapeApplicationSummary()
{
RMApp app = Org.Mockito.Mockito.Mock <RMAppImpl>();
Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(ApplicationId.NewInstance
(100L, 1));
Org.Mockito.Mockito.When(app.GetName()).ThenReturn("Multiline\n\n\r\rAppName");
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("Multiline\n\n\r\rUserName");
Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("Multiline\n\n\r\rQueueName");
Org.Mockito.Mockito.When(app.GetState()).ThenReturn(RMAppState.Running);
Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("MAPREDUCE");
RMAppMetrics metrics = new RMAppMetrics(Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1234, 56), 10, 1, 16384, 64);
Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(metrics);
RMAppManager.ApplicationSummary.SummaryBuilder summary = RMAppManager.ApplicationSummary
.CreateAppSummary(app);
string msg = summary.ToString();
Log.Info("summary: " + msg);
NUnit.Framework.Assert.IsFalse(msg.Contains("\n"));
NUnit.Framework.Assert.IsFalse(msg.Contains("\r"));
string escaped = "\\n\\n\\r\\r";
NUnit.Framework.Assert.IsTrue(msg.Contains("Multiline" + escaped + "AppName"));
NUnit.Framework.Assert.IsTrue(msg.Contains("Multiline" + escaped + "UserName"));
NUnit.Framework.Assert.IsTrue(msg.Contains("Multiline" + escaped + "QueueName"));
NUnit.Framework.Assert.IsTrue(msg.Contains("memorySeconds=16384"));
NUnit.Framework.Assert.IsTrue(msg.Contains("vcoreSeconds=64"));
NUnit.Framework.Assert.IsTrue(msg.Contains("preemptedAMContainers=1"));
NUnit.Framework.Assert.IsTrue(msg.Contains("preemptedNonAMContainers=10"));
NUnit.Framework.Assert.IsTrue(msg.Contains("preemptedResources=<memory:1234\\, vCores:56>"
));
NUnit.Framework.Assert.IsTrue(msg.Contains("applicationType=MAPREDUCE"));
}
private static RMApp CreateRMApp(ApplicationId appId)
{
RMApp app = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(appId);
Org.Mockito.Mockito.When(app.GetName()).ThenReturn("test app");
Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("test app type");
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("test user");
Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("test queue");
Org.Mockito.Mockito.When(app.GetSubmitTime()).ThenReturn(0L);
Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(1L);
Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(2L);
Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder("test diagnostics info"
));
Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
.Undefined);
return(app);
}
public virtual void TestAppBlockRenderWithNullCurrentAppAttempt()
{
ApplicationId AppId = ApplicationId.NewInstance(1234L, 0);
Injector injector;
// init app
RMApp app = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(app.GetTrackingUrl()).ThenReturn("http://host:123");
Org.Mockito.Mockito.When(app.GetState()).ThenReturn(RMAppState.Failed);
Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(AppId);
Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("Type");
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("user");
Org.Mockito.Mockito.When(app.GetName()).ThenReturn("Name");
Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("queue");
Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder());
Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
.Failed);
Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
.Failed);
Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(0L);
Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(0L);
Org.Mockito.Mockito.When(app.CreateApplicationState()).ThenReturn(YarnApplicationState
.Failed);
RMAppMetrics appMetrics = new RMAppMetrics(Resource.NewInstance(0, 0), 0, 0, 0, 0
);
Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(appMetrics);
// initialize RM Context, and create RMApp, without creating RMAppAttempt
RMContext rmContext = TestRMWebApp.MockRMContext(15, 1, 2, 8);
rmContext.GetRMApps()[AppId] = app;
injector = WebAppTests.CreateMockInjector <RMContext>(rmContext, new _Module_76(rmContext
));
AppBlock instance = injector.GetInstance <AppBlock>();
instance.Set(YarnWebParams.ApplicationId, AppId.ToString());
instance.Render();
}
public AppInfo(ResourceManager rm, RMApp app, bool hasAccess, string schemePrefix
)
{
// these are ok for any user to see
// these are only allowed if acls allow
// preemption info fields
// JAXB needs this
this.schemePrefix = schemePrefix;
if (app != null)
{
string trackingUrl = app.GetTrackingUrl();
this.state = app.CreateApplicationState();
this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.IsEmpty() || YarnApplicationState
.New == this.state || YarnApplicationState.NewSaving == this.state || YarnApplicationState
.Submitted == this.state || YarnApplicationState.Accepted == this.state;
this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app.GetFinishTime(
) == 0 ? "ApplicationMaster" : "History");
if (!trackingUrlIsNotReady)
{
this.trackingUrl = WebAppUtils.GetURLWithScheme(schemePrefix, trackingUrl);
this.trackingUrlPretty = this.trackingUrl;
}
else
{
this.trackingUrlPretty = "UNASSIGNED";
}
this.applicationId = app.GetApplicationId();
this.applicationType = app.GetApplicationType();
this.appIdNum = app.GetApplicationId().GetId().ToString();
this.id = app.GetApplicationId().ToString();
this.user = app.GetUser().ToString();
this.name = app.GetName().ToString();
this.queue = app.GetQueue().ToString();
this.progress = app.GetProgress() * 100;
this.diagnostics = app.GetDiagnostics().ToString();
if (diagnostics == null || diagnostics.IsEmpty())
{
this.diagnostics = string.Empty;
}
if (app.GetApplicationTags() != null && !app.GetApplicationTags().IsEmpty())
{
this.applicationTags = Joiner.On(',').Join(app.GetApplicationTags());
}
this.finalStatus = app.GetFinalApplicationStatus();
this.clusterId = ResourceManager.GetClusterTimeStamp();
if (hasAccess)
{
this.startedTime = app.GetStartTime();
this.finishedTime = app.GetFinishTime();
this.elapsedTime = Times.Elapsed(app.GetStartTime(), app.GetFinishTime());
RMAppAttempt attempt = app.GetCurrentAppAttempt();
if (attempt != null)
{
Container masterContainer = attempt.GetMasterContainer();
if (masterContainer != null)
{
this.amContainerLogsExist = true;
this.amContainerLogs = WebAppUtils.GetRunningLogURL(schemePrefix + masterContainer
.GetNodeHttpAddress(), ConverterUtils.ToString(masterContainer.GetId()), app.GetUser
());
this.amHostHttpAddress = masterContainer.GetNodeHttpAddress();
}
ApplicationResourceUsageReport resourceReport = attempt.GetApplicationResourceUsageReport
();
if (resourceReport != null)
{
Resource usedResources = resourceReport.GetUsedResources();
allocatedMB = usedResources.GetMemory();
allocatedVCores = usedResources.GetVirtualCores();
runningContainers = resourceReport.GetNumUsedContainers();
}
resourceRequests = ((AbstractYarnScheduler)rm.GetRMContext().GetScheduler()).GetPendingResourceRequestsForAttempt
(attempt.GetAppAttemptId());
}
}
// copy preemption info fields
RMAppMetrics appMetrics = app.GetRMAppMetrics();
numAMContainerPreempted = appMetrics.GetNumAMContainersPreempted();
preemptedResourceMB = appMetrics.GetResourcePreempted().GetMemory();
numNonAMContainerPreempted = appMetrics.GetNumNonAMContainersPreempted();
preemptedResourceVCores = appMetrics.GetResourcePreempted().GetVirtualCores();
memorySeconds = appMetrics.GetMemorySeconds();
vcoreSeconds = appMetrics.GetVcoreSeconds();
}
}
public virtual void AppCreated(RMApp app, long createdTime)
{
if (publishSystemMetrics)
{
dispatcher.GetEventHandler().Handle(new ApplicationCreatedEvent(app.GetApplicationId
(), app.GetName(), app.GetApplicationType(), app.GetUser(), app.GetQueue(), app.
GetSubmitTime(), createdTime));
}
}
public virtual void ApplicationStarted(RMApp app)
{
if (historyServiceEnabled)
{
dispatcher.GetEventHandler().Handle(new WritingApplicationStartEvent(app.GetApplicationId
(), ApplicationStartData.NewInstance(app.GetApplicationId(), app.GetName(), app.
GetApplicationType(), app.GetQueue(), app.GetUser(), app.GetSubmitTime(), app.GetStartTime
())));
}
}
/// <summary>create a summary of the application's runtime.</summary>
/// <param name="app">
///
/// <see cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Rmapp.RMApp"/>
/// whose summary is to be created, cannot
/// be <code>null</code>.
/// </param>
public static RMAppManager.ApplicationSummary.SummaryBuilder CreateAppSummary(RMApp
app)
{
string trackingUrl = "N/A";
string host = "N/A";
RMAppAttempt attempt = app.GetCurrentAppAttempt();
if (attempt != null)
{
trackingUrl = attempt.GetTrackingUrl();
host = attempt.GetHost();
}
RMAppMetrics metrics = app.GetRMAppMetrics();
RMAppManager.ApplicationSummary.SummaryBuilder summary = new RMAppManager.ApplicationSummary.SummaryBuilder
().Add("appId", app.GetApplicationId()).Add("name", app.GetName()).Add("user", app
.GetUser()).Add("queue", app.GetQueue()).Add("state", app.GetState()).Add("trackingUrl"
, trackingUrl).Add("appMasterHost", host).Add("startTime", app.GetStartTime()).Add
("finishTime", app.GetFinishTime()).Add("finalStatus", app.GetFinalApplicationStatus
()).Add("memorySeconds", metrics.GetMemorySeconds()).Add("vcoreSeconds", metrics
.GetVcoreSeconds()).Add("preemptedAMContainers", metrics.GetNumAMContainersPreempted
()).Add("preemptedNonAMContainers", metrics.GetNumNonAMContainersPreempted()).Add
("preemptedResources", metrics.GetResourcePreempted()).Add("applicationType", app
.GetApplicationType());
return(summary);
}
/// <exception cref="System.Exception"/>
public virtual void TestPublishApplicationMetrics()
{
for (int i = 1; i <= 2; ++i)
{
ApplicationId appId = ApplicationId.NewInstance(0, i);
RMApp app = CreateRMApp(appId);
metricsPublisher.AppCreated(app, app.GetStartTime());
metricsPublisher.AppFinished(app, RMAppState.Finished, app.GetFinishTime());
if (i == 1)
{
metricsPublisher.AppACLsUpdated(app, "uers1,user2", 4L);
}
else
{
// in case user doesn't specify the ACLs
metricsPublisher.AppACLsUpdated(app, null, 4L);
}
TimelineEntity entity = null;
do
{
entity = store.GetEntity(appId.ToString(), ApplicationMetricsConstants.EntityType
, EnumSet.AllOf <TimelineReader.Field>());
}while (entity == null || entity.GetEvents().Count < 3);
// ensure three events are both published before leaving the loop
// verify all the fields
NUnit.Framework.Assert.AreEqual(ApplicationMetricsConstants.EntityType, entity.GetEntityType
());
NUnit.Framework.Assert.AreEqual(app.GetApplicationId().ToString(), entity.GetEntityId
());
NUnit.Framework.Assert.AreEqual(app.GetName(), entity.GetOtherInfo()[ApplicationMetricsConstants
.NameEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetQueue(), entity.GetOtherInfo()[ApplicationMetricsConstants
.QueueEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetUser(), entity.GetOtherInfo()[ApplicationMetricsConstants
.UserEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetApplicationType(), entity.GetOtherInfo()[ApplicationMetricsConstants
.TypeEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetSubmitTime(), entity.GetOtherInfo()[ApplicationMetricsConstants
.SubmittedTimeEntityInfo]);
if (i == 1)
{
NUnit.Framework.Assert.AreEqual("uers1,user2", entity.GetOtherInfo()[ApplicationMetricsConstants
.AppViewAclsEntityInfo]);
}
else
{
NUnit.Framework.Assert.AreEqual(string.Empty, entity.GetOtherInfo()[ApplicationMetricsConstants
.AppViewAclsEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetRMAppMetrics().GetMemorySeconds(), long.Parse
(entity.GetOtherInfo()[ApplicationMetricsConstants.AppMemMetrics].ToString()));
NUnit.Framework.Assert.AreEqual(app.GetRMAppMetrics().GetVcoreSeconds(), long.Parse
(entity.GetOtherInfo()[ApplicationMetricsConstants.AppCpuMetrics].ToString()));
}
bool hasCreatedEvent = false;
bool hasFinishedEvent = false;
bool hasACLsUpdatedEvent = false;
foreach (TimelineEvent @event in entity.GetEvents())
{
if (@event.GetEventType().Equals(ApplicationMetricsConstants.CreatedEventType))
{
hasCreatedEvent = true;
NUnit.Framework.Assert.AreEqual(app.GetStartTime(), @event.GetTimestamp());
}
else
{
if (@event.GetEventType().Equals(ApplicationMetricsConstants.FinishedEventType))
{
hasFinishedEvent = true;
NUnit.Framework.Assert.AreEqual(app.GetFinishTime(), @event.GetTimestamp());
NUnit.Framework.Assert.AreEqual(app.GetDiagnostics().ToString(), @event.GetEventInfo
()[ApplicationMetricsConstants.DiagnosticsInfoEventInfo]);
NUnit.Framework.Assert.AreEqual(app.GetFinalApplicationStatus().ToString(), @event
.GetEventInfo()[ApplicationMetricsConstants.FinalStatusEventInfo]);
NUnit.Framework.Assert.AreEqual(YarnApplicationState.Finished.ToString(), @event.
GetEventInfo()[ApplicationMetricsConstants.StateEventInfo]);
}
else
{
if (@event.GetEventType().Equals(ApplicationMetricsConstants.AclsUpdatedEventType
))
{
hasACLsUpdatedEvent = true;
NUnit.Framework.Assert.AreEqual(4L, @event.GetTimestamp());
}
}
}
}
NUnit.Framework.Assert.IsTrue(hasCreatedEvent && hasFinishedEvent && hasACLsUpdatedEvent
);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestRMAppSubmitDuplicateApplicationId()
{
ApplicationId appId = MockApps.NewAppID(0);
asContext.SetApplicationId(appId);
RMApp appOrig = rmContext.GetRMApps()[appId];
NUnit.Framework.Assert.IsTrue("app name matches but shouldn't", "testApp1" != appOrig
.GetName());
// our testApp1 should be rejected and original app with same id should be left in place
try
{
appMonitor.SubmitApplication(asContext, "test");
NUnit.Framework.Assert.Fail("Exception is expected when applicationId is duplicate."
);
}
catch (YarnException e)
{
NUnit.Framework.Assert.IsTrue("The thrown exception is not the expectd one.", e.Message
.Contains("Cannot add a duplicate!"));
}
// make sure original app didn't get removed
RMApp app = rmContext.GetRMApps()[appId];
NUnit.Framework.Assert.IsNotNull("app is null", app);
NUnit.Framework.Assert.AreEqual("app id doesn't match", appId, app.GetApplicationId
());
NUnit.Framework.Assert.AreEqual("app state doesn't match", RMAppState.Finished, app
.GetState());
}