private static RMApp CreateRMApp(ApplicationId appId)
{
RMApp app = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(appId);
Org.Mockito.Mockito.When(app.GetName()).ThenReturn("test app");
Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("test app type");
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("test user");
Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("test queue");
Org.Mockito.Mockito.When(app.GetSubmitTime()).ThenReturn(int.MaxValue + 1L);
Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(int.MaxValue + 2L);
Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(int.MaxValue + 3L);
Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder("test diagnostics info"
));
RMAppAttempt appAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(appAttempt.GetAppAttemptId()).ThenReturn(ApplicationAttemptId
.NewInstance(appId, 1));
Org.Mockito.Mockito.When(app.GetCurrentAppAttempt()).ThenReturn(appAttempt);
Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
.Undefined);
Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(new RMAppMetrics(null,
0, 0, int.MaxValue, long.MaxValue));
return(app);
}
public virtual void TestEscapeApplicationSummary()
{
RMApp app = Org.Mockito.Mockito.Mock <RMAppImpl>();
Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(ApplicationId.NewInstance
(100L, 1));
Org.Mockito.Mockito.When(app.GetName()).ThenReturn("Multiline\n\n\r\rAppName");
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("Multiline\n\n\r\rUserName");
Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("Multiline\n\n\r\rQueueName");
Org.Mockito.Mockito.When(app.GetState()).ThenReturn(RMAppState.Running);
Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("MAPREDUCE");
RMAppMetrics metrics = new RMAppMetrics(Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1234, 56), 10, 1, 16384, 64);
Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(metrics);
RMAppManager.ApplicationSummary.SummaryBuilder summary = RMAppManager.ApplicationSummary
.CreateAppSummary(app);
string msg = summary.ToString();
Log.Info("summary: " + msg);
NUnit.Framework.Assert.IsFalse(msg.Contains("\n"));
NUnit.Framework.Assert.IsFalse(msg.Contains("\r"));
string escaped = "\\n\\n\\r\\r";
NUnit.Framework.Assert.IsTrue(msg.Contains("Multiline" + escaped + "AppName"));
NUnit.Framework.Assert.IsTrue(msg.Contains("Multiline" + escaped + "UserName"));
NUnit.Framework.Assert.IsTrue(msg.Contains("Multiline" + escaped + "QueueName"));
NUnit.Framework.Assert.IsTrue(msg.Contains("memorySeconds=16384"));
NUnit.Framework.Assert.IsTrue(msg.Contains("vcoreSeconds=64"));
NUnit.Framework.Assert.IsTrue(msg.Contains("preemptedAMContainers=1"));
NUnit.Framework.Assert.IsTrue(msg.Contains("preemptedNonAMContainers=10"));
NUnit.Framework.Assert.IsTrue(msg.Contains("preemptedResources=<memory:1234\\, vCores:56>"
));
NUnit.Framework.Assert.IsTrue(msg.Contains("applicationType=MAPREDUCE"));
}
protected internal virtual void WriteAuditLog(ApplicationId appId)
{
RMApp app = rmContext.GetRMApps()[appId];
string operation = "UNKONWN";
bool success = false;
switch (app.GetState())
{
case RMAppState.Failed:
{
operation = RMAuditLogger.AuditConstants.FinishFailedApp;
break;
}
case RMAppState.Finished:
{
operation = RMAuditLogger.AuditConstants.FinishSuccessApp;
success = true;
break;
}
case RMAppState.Killed:
{
operation = RMAuditLogger.AuditConstants.FinishKilledApp;
success = true;
break;
}
default:
{
break;
}
}
if (success)
{
RMAuditLogger.LogSuccess(app.GetUser(), operation, "RMAppManager", app.GetApplicationId
());
}
else
{
StringBuilder diag = app.GetDiagnostics();
string msg = diag == null ? null : diag.ToString();
RMAuditLogger.LogFailure(app.GetUser(), operation, msg, "RMAppManager", "App failed with state: "
+ app.GetState(), appId);
}
}
public virtual void TestDelegationTokenAuth()
{
string token = GetDelegationToken("test");
ApplicationSubmissionContextInfo app = new ApplicationSubmissionContextInfo();
string appid = "application_123_0";
app.SetApplicationId(appid);
string requestBody = GetMarshalledAppInfo(app);
Uri url = new Uri("http://localhost:8088/ws/v1/cluster/apps");
HttpURLConnection conn = (HttpURLConnection)url.OpenConnection();
SetupConn(conn, "POST", "application/xml", requestBody);
// this should fail with unauthorized because only
// auth is kerberos or delegation token
try
{
conn.GetInputStream();
NUnit.Framework.Assert.Fail("we should not be here");
}
catch (IOException)
{
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.Unauthorized.GetStatusCode(
), conn.GetResponseCode());
}
conn = (HttpURLConnection)url.OpenConnection();
conn.SetRequestProperty(delegationTokenHeader, token);
SetupConn(conn, "POST", MediaType.ApplicationXml, requestBody);
// this should not fail
try
{
conn.GetInputStream();
}
catch (IOException)
{
InputStream errorStream = conn.GetErrorStream();
string error = string.Empty;
BufferedReader reader = null;
reader = new BufferedReader(new InputStreamReader(errorStream, "UTF8"));
for (string line; (line = reader.ReadLine()) != null;)
{
error += line;
}
reader.Close();
errorStream.Close();
NUnit.Framework.Assert.Fail("Response " + conn.GetResponseCode() + "; " + error);
}
bool appExists = rm.GetRMContext().GetRMApps().Contains(ConverterUtils.ToApplicationId
(appid));
NUnit.Framework.Assert.IsTrue(appExists);
RMApp actualApp = rm.GetRMContext().GetRMApps()[ConverterUtils.ToApplicationId(appid
)];
string owner = actualApp.GetUser();
NUnit.Framework.Assert.AreEqual("client", owner);
}
/// <exception cref="System.Exception"/>
private void TestAnonymousSimpleUser()
{
ApplicationSubmissionContextInfo app = new ApplicationSubmissionContextInfo();
string appid = "application_123_0";
app.SetApplicationId(appid);
string requestBody = TestRMWebServicesDelegationTokenAuthentication.GetMarshalledAppInfo
(app);
Uri url = new Uri("http://localhost:8088/ws/v1/cluster/apps");
HttpURLConnection conn = (HttpURLConnection)url.OpenConnection();
TestRMWebServicesDelegationTokenAuthentication.SetupConn(conn, "POST", "application/xml"
, requestBody);
conn.GetInputStream();
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.Accepted.GetStatusCode(), conn
.GetResponseCode());
bool appExists = rm.GetRMContext().GetRMApps().Contains(ConverterUtils.ToApplicationId
(appid));
NUnit.Framework.Assert.IsTrue(appExists);
RMApp actualApp = rm.GetRMContext().GetRMApps()[ConverterUtils.ToApplicationId(appid
)];
string owner = actualApp.GetUser();
NUnit.Framework.Assert.AreEqual(rm.GetConfig().Get(CommonConfigurationKeys.HadoopHttpStaticUser
, CommonConfigurationKeys.DefaultHadoopHttpStaticUser), owner);
appid = "application_123_1";
app.SetApplicationId(appid);
requestBody = TestRMWebServicesDelegationTokenAuthentication.GetMarshalledAppInfo
(app);
url = new Uri("http://localhost:8088/ws/v1/cluster/apps?user.name=client");
conn = (HttpURLConnection)url.OpenConnection();
TestRMWebServicesDelegationTokenAuthentication.SetupConn(conn, "POST", MediaType.
ApplicationXml, requestBody);
conn.GetInputStream();
appExists = rm.GetRMContext().GetRMApps().Contains(ConverterUtils.ToApplicationId
(appid));
NUnit.Framework.Assert.IsTrue(appExists);
actualApp = rm.GetRMContext().GetRMApps()[ConverterUtils.ToApplicationId(appid)];
owner = actualApp.GetUser();
NUnit.Framework.Assert.AreEqual("client", owner);
}
/// <exception cref="System.Exception"/>
protected internal virtual RMApp StoreApp(RMStateStore store, ApplicationId appId
, long submitTime, long startTime)
{
ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();
context.SetApplicationId(appId);
RMApp mockApp = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(mockApp.GetApplicationId()).ThenReturn(appId);
Org.Mockito.Mockito.When(mockApp.GetSubmitTime()).ThenReturn(submitTime);
Org.Mockito.Mockito.When(mockApp.GetStartTime()).ThenReturn(startTime);
Org.Mockito.Mockito.When(mockApp.GetApplicationSubmissionContext()).ThenReturn(context
);
Org.Mockito.Mockito.When(mockApp.GetUser()).ThenReturn("test");
store.StoreNewApplication(mockApp);
return(mockApp);
}
private static RMApp CreateRMApp(ApplicationId appId)
{
RMApp app = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(appId);
Org.Mockito.Mockito.When(app.GetName()).ThenReturn("test app");
Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("test app type");
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("test user");
Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("test queue");
Org.Mockito.Mockito.When(app.GetSubmitTime()).ThenReturn(0L);
Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(1L);
Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(2L);
Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder("test diagnostics info"
));
Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
.Undefined);
return(app);
}
public virtual void TestAppBlockRenderWithNullCurrentAppAttempt()
{
ApplicationId AppId = ApplicationId.NewInstance(1234L, 0);
Injector injector;
// init app
RMApp app = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(app.GetTrackingUrl()).ThenReturn("http://host:123");
Org.Mockito.Mockito.When(app.GetState()).ThenReturn(RMAppState.Failed);
Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(AppId);
Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("Type");
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("user");
Org.Mockito.Mockito.When(app.GetName()).ThenReturn("Name");
Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("queue");
Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder());
Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
.Failed);
Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
.Failed);
Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(0L);
Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(0L);
Org.Mockito.Mockito.When(app.CreateApplicationState()).ThenReturn(YarnApplicationState
.Failed);
RMAppMetrics appMetrics = new RMAppMetrics(Resource.NewInstance(0, 0), 0, 0, 0, 0
);
Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(appMetrics);
// initialize RM Context, and create RMApp, without creating RMAppAttempt
RMContext rmContext = TestRMWebApp.MockRMContext(15, 1, 2, 8);
rmContext.GetRMApps()[AppId] = app;
injector = WebAppTests.CreateMockInjector <RMContext>(rmContext, new _Module_76(rmContext
));
AppBlock instance = injector.GetInstance <AppBlock>();
instance.Set(YarnWebParams.ApplicationId, AppId.ToString());
instance.Render();
}
public AppInfo(ResourceManager rm, RMApp app, bool hasAccess, string schemePrefix
)
{
// these are ok for any user to see
// these are only allowed if acls allow
// preemption info fields
// JAXB needs this
this.schemePrefix = schemePrefix;
if (app != null)
{
string trackingUrl = app.GetTrackingUrl();
this.state = app.CreateApplicationState();
this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.IsEmpty() || YarnApplicationState
.New == this.state || YarnApplicationState.NewSaving == this.state || YarnApplicationState
.Submitted == this.state || YarnApplicationState.Accepted == this.state;
this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app.GetFinishTime(
) == 0 ? "ApplicationMaster" : "History");
if (!trackingUrlIsNotReady)
{
this.trackingUrl = WebAppUtils.GetURLWithScheme(schemePrefix, trackingUrl);
this.trackingUrlPretty = this.trackingUrl;
}
else
{
this.trackingUrlPretty = "UNASSIGNED";
}
this.applicationId = app.GetApplicationId();
this.applicationType = app.GetApplicationType();
this.appIdNum = app.GetApplicationId().GetId().ToString();
this.id = app.GetApplicationId().ToString();
this.user = app.GetUser().ToString();
this.name = app.GetName().ToString();
this.queue = app.GetQueue().ToString();
this.progress = app.GetProgress() * 100;
this.diagnostics = app.GetDiagnostics().ToString();
if (diagnostics == null || diagnostics.IsEmpty())
{
this.diagnostics = string.Empty;
}
if (app.GetApplicationTags() != null && !app.GetApplicationTags().IsEmpty())
{
this.applicationTags = Joiner.On(',').Join(app.GetApplicationTags());
}
this.finalStatus = app.GetFinalApplicationStatus();
this.clusterId = ResourceManager.GetClusterTimeStamp();
if (hasAccess)
{
this.startedTime = app.GetStartTime();
this.finishedTime = app.GetFinishTime();
this.elapsedTime = Times.Elapsed(app.GetStartTime(), app.GetFinishTime());
RMAppAttempt attempt = app.GetCurrentAppAttempt();
if (attempt != null)
{
Container masterContainer = attempt.GetMasterContainer();
if (masterContainer != null)
{
this.amContainerLogsExist = true;
this.amContainerLogs = WebAppUtils.GetRunningLogURL(schemePrefix + masterContainer
.GetNodeHttpAddress(), ConverterUtils.ToString(masterContainer.GetId()), app.GetUser
());
this.amHostHttpAddress = masterContainer.GetNodeHttpAddress();
}
ApplicationResourceUsageReport resourceReport = attempt.GetApplicationResourceUsageReport
();
if (resourceReport != null)
{
Resource usedResources = resourceReport.GetUsedResources();
allocatedMB = usedResources.GetMemory();
allocatedVCores = usedResources.GetVirtualCores();
runningContainers = resourceReport.GetNumUsedContainers();
}
resourceRequests = ((AbstractYarnScheduler)rm.GetRMContext().GetScheduler()).GetPendingResourceRequestsForAttempt
(attempt.GetAppAttemptId());
}
}
// copy preemption info fields
RMAppMetrics appMetrics = app.GetRMAppMetrics();
numAMContainerPreempted = appMetrics.GetNumAMContainersPreempted();
preemptedResourceMB = appMetrics.GetResourcePreempted().GetMemory();
numNonAMContainerPreempted = appMetrics.GetNumNonAMContainersPreempted();
preemptedResourceVCores = appMetrics.GetResourcePreempted().GetVirtualCores();
memorySeconds = appMetrics.GetMemorySeconds();
vcoreSeconds = appMetrics.GetVcoreSeconds();
}
}
public virtual void AppCreated(RMApp app, long createdTime)
{
if (publishSystemMetrics)
{
dispatcher.GetEventHandler().Handle(new ApplicationCreatedEvent(app.GetApplicationId
(), app.GetName(), app.GetApplicationType(), app.GetUser(), app.GetQueue(), app.
GetSubmitTime(), createdTime));
}
}
/// <exception cref="System.Exception"/>
public virtual void TestPublishApplicationMetrics()
{
for (int i = 1; i <= 2; ++i)
{
ApplicationId appId = ApplicationId.NewInstance(0, i);
RMApp app = CreateRMApp(appId);
metricsPublisher.AppCreated(app, app.GetStartTime());
metricsPublisher.AppFinished(app, RMAppState.Finished, app.GetFinishTime());
if (i == 1)
{
metricsPublisher.AppACLsUpdated(app, "uers1,user2", 4L);
}
else
{
// in case user doesn't specify the ACLs
metricsPublisher.AppACLsUpdated(app, null, 4L);
}
TimelineEntity entity = null;
do
{
entity = store.GetEntity(appId.ToString(), ApplicationMetricsConstants.EntityType
, EnumSet.AllOf <TimelineReader.Field>());
}while (entity == null || entity.GetEvents().Count < 3);
// ensure three events are both published before leaving the loop
// verify all the fields
NUnit.Framework.Assert.AreEqual(ApplicationMetricsConstants.EntityType, entity.GetEntityType
());
NUnit.Framework.Assert.AreEqual(app.GetApplicationId().ToString(), entity.GetEntityId
());
NUnit.Framework.Assert.AreEqual(app.GetName(), entity.GetOtherInfo()[ApplicationMetricsConstants
.NameEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetQueue(), entity.GetOtherInfo()[ApplicationMetricsConstants
.QueueEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetUser(), entity.GetOtherInfo()[ApplicationMetricsConstants
.UserEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetApplicationType(), entity.GetOtherInfo()[ApplicationMetricsConstants
.TypeEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetSubmitTime(), entity.GetOtherInfo()[ApplicationMetricsConstants
.SubmittedTimeEntityInfo]);
if (i == 1)
{
NUnit.Framework.Assert.AreEqual("uers1,user2", entity.GetOtherInfo()[ApplicationMetricsConstants
.AppViewAclsEntityInfo]);
}
else
{
NUnit.Framework.Assert.AreEqual(string.Empty, entity.GetOtherInfo()[ApplicationMetricsConstants
.AppViewAclsEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetRMAppMetrics().GetMemorySeconds(), long.Parse
(entity.GetOtherInfo()[ApplicationMetricsConstants.AppMemMetrics].ToString()));
NUnit.Framework.Assert.AreEqual(app.GetRMAppMetrics().GetVcoreSeconds(), long.Parse
(entity.GetOtherInfo()[ApplicationMetricsConstants.AppCpuMetrics].ToString()));
}
bool hasCreatedEvent = false;
bool hasFinishedEvent = false;
bool hasACLsUpdatedEvent = false;
foreach (TimelineEvent @event in entity.GetEvents())
{
if (@event.GetEventType().Equals(ApplicationMetricsConstants.CreatedEventType))
{
hasCreatedEvent = true;
NUnit.Framework.Assert.AreEqual(app.GetStartTime(), @event.GetTimestamp());
}
else
{
if (@event.GetEventType().Equals(ApplicationMetricsConstants.FinishedEventType))
{
hasFinishedEvent = true;
NUnit.Framework.Assert.AreEqual(app.GetFinishTime(), @event.GetTimestamp());
NUnit.Framework.Assert.AreEqual(app.GetDiagnostics().ToString(), @event.GetEventInfo
()[ApplicationMetricsConstants.DiagnosticsInfoEventInfo]);
NUnit.Framework.Assert.AreEqual(app.GetFinalApplicationStatus().ToString(), @event
.GetEventInfo()[ApplicationMetricsConstants.FinalStatusEventInfo]);
NUnit.Framework.Assert.AreEqual(YarnApplicationState.Finished.ToString(), @event.
GetEventInfo()[ApplicationMetricsConstants.StateEventInfo]);
}
else
{
if (@event.GetEventType().Equals(ApplicationMetricsConstants.AclsUpdatedEventType
))
{
hasACLsUpdatedEvent = true;
NUnit.Framework.Assert.AreEqual(4L, @event.GetTimestamp());
}
}
}
}
NUnit.Framework.Assert.IsTrue(hasCreatedEvent && hasFinishedEvent && hasACLsUpdatedEvent
);
}
}
protected override void GenerateApplicationTable(HtmlBlock.Block html, UserGroupInformation
callerUGI, ICollection <ApplicationAttemptReport> attempts)
{
// Application Attempt Table
Hamlet.TBODY <Hamlet.TABLE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > tbody = html
.Table("#attempts").Thead().Tr().Th(".id", "Attempt ID").Th(".started", "Started"
).Th(".node", "Node").Th(".logs", "Logs").Th(".blacklistednodes", "Blacklisted Nodes"
).().().Tbody();
RMApp rmApp = this.rm.GetRMContext().GetRMApps()[this.appID];
if (rmApp == null)
{
return;
}
StringBuilder attemptsTableData = new StringBuilder("[\n");
foreach (ApplicationAttemptReport appAttemptReport in attempts)
{
RMAppAttempt rmAppAttempt = rmApp.GetRMAppAttempt(appAttemptReport.GetApplicationAttemptId
());
if (rmAppAttempt == null)
{
continue;
}
AppAttemptInfo attemptInfo = new AppAttemptInfo(this.rm, rmAppAttempt, rmApp.GetUser
(), WebAppUtils.GetHttpSchemePrefix(conf));
string blacklistedNodesCount = "N/A";
ICollection <string> nodes = RMAppAttemptBlock.GetBlacklistedNodes(rm, rmAppAttempt
.GetAppAttemptId());
if (nodes != null)
{
blacklistedNodesCount = nodes.Count.ToString();
}
string nodeLink = attemptInfo.GetNodeHttpAddress();
if (nodeLink != null)
{
nodeLink = WebAppUtils.GetHttpSchemePrefix(conf) + nodeLink;
}
string logsLink = attemptInfo.GetLogsLink();
attemptsTableData.Append("[\"<a href='").Append(Url("appattempt", rmAppAttempt.GetAppAttemptId
().ToString())).Append("'>").Append(rmAppAttempt.GetAppAttemptId().ToString()).Append
("</a>\",\"").Append(attemptInfo.GetStartTime()).Append("\",\"<a ").Append(nodeLink
== null ? "#" : "href='" + nodeLink).Append("'>").Append(nodeLink == null ? "N/A"
: StringEscapeUtils.EscapeJavaScript(StringEscapeUtils.EscapeHtml(nodeLink))).Append
("</a>\",\"<a ").Append(logsLink == null ? "#" : "href='" + logsLink).Append("'>"
).Append(logsLink == null ? "N/A" : "Logs").Append("</a>\",").Append("\"").Append
(blacklistedNodesCount).Append("\"],\n");
}
if (attemptsTableData[attemptsTableData.Length - 2] == ',')
{
attemptsTableData.Delete(attemptsTableData.Length - 2, attemptsTableData.Length -
1);
}
attemptsTableData.Append("]");
html.Script().$type("text/javascript").("var attemptsTableData=" + attemptsTableData
).();
tbody.().();
}
public virtual int Run(string[] args)
{
Log.Info("Starting ZKRMStateStorePerf ver." + version);
int numApp = ZkPerfNumAppDefault;
int numAppAttemptPerApp = ZkPerfNumAppattemptPerApp;
string hostPort = null;
bool launchLocalZK = true;
if (args.Length == 0)
{
System.Console.Error.WriteLine("Missing arguments.");
return(-1);
}
for (int i = 0; i < args.Length; i++)
{
// parse command line
if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-appsize"))
{
numApp = System.Convert.ToInt32(args[++i]);
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-appattemptsize"))
{
numAppAttemptPerApp = System.Convert.ToInt32(args[++i]);
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-hostPort"))
{
hostPort = args[++i];
launchLocalZK = false;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-workingZnode"))
{
workingZnode = args[++i];
}
else
{
System.Console.Error.WriteLine("Illegal argument: " + args[i]);
return(-1);
}
}
}
}
}
if (launchLocalZK)
{
try
{
SetUp();
}
catch (Exception e)
{
System.Console.Error.WriteLine("failed to setup. : " + e.Message);
return(-1);
}
}
InitStore(hostPort);
long submitTime = Runtime.CurrentTimeMillis();
long startTime = Runtime.CurrentTimeMillis() + 1234;
AList <ApplicationId> applicationIds = new AList <ApplicationId>();
AList <RMApp> rmApps = new AList <RMApp>();
AList <ApplicationAttemptId> attemptIds = new AList <ApplicationAttemptId>();
Dictionary <ApplicationId, ICollection <ApplicationAttemptId> > appIdsToAttemptId =
new Dictionary <ApplicationId, ICollection <ApplicationAttemptId> >();
RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
();
store.SetRMDispatcher(dispatcher);
for (int i_1 = 0; i_1 < numApp; i_1++)
{
ApplicationId appId = ApplicationId.NewInstance(clusterTimeStamp, i_1);
applicationIds.AddItem(appId);
AList <ApplicationAttemptId> attemptIdsForThisApp = new AList <ApplicationAttemptId
>();
for (int j = 0; j < numAppAttemptPerApp; j++)
{
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, j);
attemptIdsForThisApp.AddItem(attemptId);
}
appIdsToAttemptId[appId] = new LinkedHashSet(attemptIdsForThisApp);
Sharpen.Collections.AddAll(attemptIds, attemptIdsForThisApp);
}
foreach (ApplicationId appId_1 in applicationIds)
{
RMApp app = null;
try
{
app = StoreApp(store, appId_1, submitTime, startTime);
}
catch (Exception e)
{
System.Console.Error.WriteLine("failed to create Application Znode. : " + e.Message
);
return(-1);
}
WaitNotify(dispatcher);
rmApps.AddItem(app);
}
foreach (ApplicationAttemptId attemptId_1 in attemptIds)
{
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> tokenId = GenerateAMRMToken
(attemptId_1, appTokenMgr);
SecretKey clientTokenKey = clientToAMTokenMgr.CreateMasterKey(attemptId_1);
try
{
StoreAttempt(store, attemptId_1, ContainerId.NewContainerId(attemptId_1, 0L).ToString
(), tokenId, clientTokenKey, dispatcher);
}
catch (Exception e)
{
System.Console.Error.WriteLine("failed to create AppAttempt Znode. : " + e.Message
);
return(-1);
}
}
long storeStart = Runtime.CurrentTimeMillis();
try
{
store.LoadState();
}
catch (Exception e)
{
System.Console.Error.WriteLine("failed to locaState from ZKRMStateStore. : " + e.
Message);
return(-1);
}
long storeEnd = Runtime.CurrentTimeMillis();
long loadTime = storeEnd - storeStart;
string resultMsg = "ZKRMStateStore takes " + loadTime + " msec to loadState.";
Log.Info(resultMsg);
System.Console.Out.WriteLine(resultMsg);
// cleanup
try
{
foreach (RMApp app in rmApps)
{
ApplicationStateData appState = ApplicationStateData.NewInstance(app.GetSubmitTime
(), app.GetStartTime(), app.GetApplicationSubmissionContext(), app.GetUser());
ApplicationId appId = app.GetApplicationId();
IDictionary m = Org.Mockito.Mockito.Mock <IDictionary>();
Org.Mockito.Mockito.When(m.Keys).ThenReturn(appIdsToAttemptId[appId_1]);
appState.attempts = m;
store.RemoveApplicationStateInternal(appState);
}
}
catch (Exception e)
{
System.Console.Error.WriteLine("failed to cleanup. : " + e.Message);
return(-1);
}
return(0);
}
public virtual void TestFencedState()
{
TestZKRMStateStore.TestZKRMStateStoreTester zkTester = new TestZKRMStateStore.TestZKRMStateStoreTester
(this);
RMStateStore store = zkTester.GetRMStateStore();
// Move state to FENCED from ACTIVE
store.UpdateFencedState();
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
long submitTime = Runtime.CurrentTimeMillis();
long startTime = submitTime + 1000;
// Add a new app
RMApp mockApp = Org.Mockito.Mockito.Mock <RMApp>();
ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();
Org.Mockito.Mockito.When(mockApp.GetSubmitTime()).ThenReturn(submitTime);
Org.Mockito.Mockito.When(mockApp.GetStartTime()).ThenReturn(startTime);
Org.Mockito.Mockito.When(mockApp.GetApplicationSubmissionContext()).ThenReturn(context
);
Org.Mockito.Mockito.When(mockApp.GetUser()).ThenReturn("test");
store.StoreNewApplication(mockApp);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// Add a new attempt
ClientToAMTokenSecretManagerInRM clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM
();
ApplicationAttemptId attemptId = ConverterUtils.ToApplicationAttemptId("appattempt_1234567894321_0001_000001"
);
SecretKey clientTokenMasterKey = clientToAMTokenMgr.CreateMasterKey(attemptId);
RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics
>();
Container container = new ContainerPBImpl();
container.SetId(ConverterUtils.ToContainerId("container_1234567891234_0001_01_000001"
));
RMAppAttempt mockAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(mockAttempt.GetAppAttemptId()).ThenReturn(attemptId);
Org.Mockito.Mockito.When(mockAttempt.GetMasterContainer()).ThenReturn(container);
Org.Mockito.Mockito.When(mockAttempt.GetClientTokenMasterKey()).ThenReturn(clientTokenMasterKey
);
Org.Mockito.Mockito.When(mockAttempt.GetRMAppAttemptMetrics()).ThenReturn(mockRmAppAttemptMetrics
);
Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()).
ThenReturn(new AggregateAppResourceUsage(0, 0));
store.StoreNewApplicationAttempt(mockAttempt);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
long finishTime = submitTime + 1000;
// Update attempt
ApplicationAttemptStateData newAttemptState = ApplicationAttemptStateData.NewInstance
(attemptId, container, store.GetCredentialsFromAppAttempt(mockAttempt), startTime
, RMAppAttemptState.Finished, "testUrl", "test", FinalApplicationStatus.Succeeded
, 100, finishTime, 0, 0);
store.UpdateApplicationAttemptState(newAttemptState);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// Update app
ApplicationStateData appState = ApplicationStateData.NewInstance(submitTime, startTime
, context, "test");
store.UpdateApplicationState(appState);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// Remove app
store.RemoveApplication(mockApp);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// store RM delegation token;
RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier(new Text("owner1"
), new Text("renewer1"), new Text("realuser1"));
long renewDate1 = Runtime.CurrentTimeMillis();
dtId1.SetSequenceNumber(1111);
store.StoreRMDelegationToken(dtId1, renewDate1);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
store.UpdateRMDelegationToken(dtId1, renewDate1);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// remove delegation key;
store.RemoveRMDelegationToken(dtId1);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// store delegation master key;
DelegationKey key = new DelegationKey(1234, 4321, Sharpen.Runtime.GetBytesForString
("keyBytes"));
store.StoreRMDTMasterKey(key);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// remove delegation master key;
store.RemoveRMDTMasterKey(key);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// store or update AMRMToken;
store.StoreOrUpdateAMRMTokenSecretManager(null, false);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
store.Close();
}
/// <summary>create a summary of the application's runtime.</summary>
/// <param name="app">
///
/// <see cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Rmapp.RMApp"/>
/// whose summary is to be created, cannot
/// be <code>null</code>.
/// </param>
public static RMAppManager.ApplicationSummary.SummaryBuilder CreateAppSummary(RMApp
app)
{
string trackingUrl = "N/A";
string host = "N/A";
RMAppAttempt attempt = app.GetCurrentAppAttempt();
if (attempt != null)
{
trackingUrl = attempt.GetTrackingUrl();
host = attempt.GetHost();
}
RMAppMetrics metrics = app.GetRMAppMetrics();
RMAppManager.ApplicationSummary.SummaryBuilder summary = new RMAppManager.ApplicationSummary.SummaryBuilder
().Add("appId", app.GetApplicationId()).Add("name", app.GetName()).Add("user", app
.GetUser()).Add("queue", app.GetQueue()).Add("state", app.GetState()).Add("trackingUrl"
, trackingUrl).Add("appMasterHost", host).Add("startTime", app.GetStartTime()).Add
("finishTime", app.GetFinishTime()).Add("finalStatus", app.GetFinalApplicationStatus
()).Add("memorySeconds", metrics.GetMemorySeconds()).Add("vcoreSeconds", metrics
.GetVcoreSeconds()).Add("preemptedAMContainers", metrics.GetNumAMContainersPreempted
()).Add("preemptedNonAMContainers", metrics.GetNumNonAMContainersPreempted()).Add
("preemptedResources", metrics.GetResourcePreempted()).Add("applicationType", app
.GetApplicationType());
return(summary);
}
public virtual void ApplicationStarted(RMApp app)
{
if (historyServiceEnabled)
{
dispatcher.GetEventHandler().Handle(new WritingApplicationStartEvent(app.GetApplicationId
(), ApplicationStartData.NewInstance(app.GetApplicationId(), app.GetName(), app.
GetApplicationType(), app.GetQueue(), app.GetUser(), app.GetSubmitTime(), app.GetStartTime
())));
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual RegisterApplicationMasterResponse RegisterApplicationMaster(RegisterApplicationMasterRequest
request)
{
AMRMTokenIdentifier amrmTokenIdentifier = AuthorizeRequest();
ApplicationAttemptId applicationAttemptId = amrmTokenIdentifier.GetApplicationAttemptId
();
ApplicationId appID = applicationAttemptId.GetApplicationId();
ApplicationMasterService.AllocateResponseLock Lock = responseMap[applicationAttemptId
];
if (Lock == null)
{
RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appID].GetUser(), RMAuditLogger.AuditConstants
.RegisterAm, "Application doesn't exist in cache " + applicationAttemptId, "ApplicationMasterService"
, "Error in registering application master", appID, applicationAttemptId);
ThrowApplicationDoesNotExistInCacheException(applicationAttemptId);
}
// Allow only one thread in AM to do registerApp at a time.
lock (Lock)
{
AllocateResponse lastResponse = Lock.GetAllocateResponse();
if (HasApplicationMasterRegistered(applicationAttemptId))
{
string message = "Application Master is already registered : " + appID;
Log.Warn(message);
RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appID].GetUser(), RMAuditLogger.AuditConstants
.RegisterAm, string.Empty, "ApplicationMasterService", message, appID, applicationAttemptId
);
throw new InvalidApplicationMasterRequestException(message);
}
this.amLivelinessMonitor.ReceivedPing(applicationAttemptId);
RMApp app = this.rmContext.GetRMApps()[appID];
// Setting the response id to 0 to identify if the
// application master is register for the respective attemptid
lastResponse.SetResponseId(0);
Lock.SetAllocateResponse(lastResponse);
Log.Info("AM registration " + applicationAttemptId);
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptRegistrationEvent
(applicationAttemptId, request.GetHost(), request.GetRpcPort(), request.GetTrackingUrl
()));
RMAuditLogger.LogSuccess(app.GetUser(), RMAuditLogger.AuditConstants.RegisterAm,
"ApplicationMasterService", appID, applicationAttemptId);
// Pick up min/max resource from scheduler...
RegisterApplicationMasterResponse response = recordFactory.NewRecordInstance <RegisterApplicationMasterResponse
>();
response.SetMaximumResourceCapability(rScheduler.GetMaximumResourceCapability(app
.GetQueue()));
response.SetApplicationACLs(app.GetRMAppAttempt(applicationAttemptId).GetSubmissionContext
().GetAMContainerSpec().GetApplicationACLs());
response.SetQueue(app.GetQueue());
if (UserGroupInformation.IsSecurityEnabled())
{
Log.Info("Setting client token master key");
response.SetClientToAMTokenMasterKey(ByteBuffer.Wrap(rmContext.GetClientToAMTokenSecretManager
().GetMasterKey(applicationAttemptId).GetEncoded()));
}
// For work-preserving AM restart, retrieve previous attempts' containers
// and corresponding NM tokens.
if (app.GetApplicationSubmissionContext().GetKeepContainersAcrossApplicationAttempts
())
{
IList <Container> transferredContainers = ((AbstractYarnScheduler)rScheduler).GetTransferredContainers
(applicationAttemptId);
if (!transferredContainers.IsEmpty())
{
response.SetContainersFromPreviousAttempts(transferredContainers);
IList <NMToken> nmTokens = new AList <NMToken>();
foreach (Container container in transferredContainers)
{
try
{
NMToken token = rmContext.GetNMTokenSecretManager().CreateAndGetNMToken(app.GetUser
(), applicationAttemptId, container);
if (null != token)
{
nmTokens.AddItem(token);
}
}
catch (ArgumentException e)
{
// if it's a DNS issue, throw UnknowHostException directly and
// that
// will be automatically retried by RMProxy in RPC layer.
if (e.InnerException is UnknownHostException)
{
throw (UnknownHostException)e.InnerException;
}
}
}
response.SetNMTokensFromPreviousAttempts(nmTokens);
Log.Info("Application " + appID + " retrieved " + transferredContainers.Count + " containers from previous"
+ " attempts and " + nmTokens.Count + " NM tokens.");
}
}
response.SetSchedulerResourceTypes(rScheduler.GetSchedulingResourceTypes());
return(response);
}
}
/// <exception cref="System.Exception"/>
internal virtual void TestRMAppStateStore(RMStateStoreTestBase.RMStateStoreHelper
stateStoreHelper, RMStateStoreTestBase.StoreStateVerifier verifier)
{
long submitTime = Runtime.CurrentTimeMillis();
long startTime = Runtime.CurrentTimeMillis() + 1234;
Configuration conf = new YarnConfiguration();
RMStateStore store = stateStoreHelper.GetRMStateStore();
RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
();
store.SetRMDispatcher(dispatcher);
RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();
Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store);
AMRMTokenSecretManager appTokenMgr = Org.Mockito.Mockito.Spy(new AMRMTokenSecretManager
(conf, rmContext));
MasterKeyData masterKeyData = appTokenMgr.CreateNewMasterKey();
Org.Mockito.Mockito.When(appTokenMgr.GetMasterKey()).ThenReturn(masterKeyData);
ClientToAMTokenSecretManagerInRM clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM
();
ApplicationAttemptId attemptId1 = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0001_000001"
);
ApplicationId appId1 = attemptId1.GetApplicationId();
StoreApp(store, appId1, submitTime, startTime);
verifier.AfterStoreApp(store, appId1);
// create application token and client token key for attempt1
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken1 = GenerateAMRMToken
(attemptId1, appTokenMgr);
SecretKey clientTokenKey1 = clientToAMTokenMgr.CreateMasterKey(attemptId1);
ContainerId containerId1 = StoreAttempt(store, attemptId1, "container_1352994193343_0001_01_000001"
, appAttemptToken1, clientTokenKey1, dispatcher);
string appAttemptIdStr2 = "appattempt_1352994193343_0001_000002";
ApplicationAttemptId attemptId2 = ConverterUtils.ToApplicationAttemptId(appAttemptIdStr2
);
// create application token and client token key for attempt2
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken2 = GenerateAMRMToken
(attemptId2, appTokenMgr);
SecretKey clientTokenKey2 = clientToAMTokenMgr.CreateMasterKey(attemptId2);
ContainerId containerId2 = StoreAttempt(store, attemptId2, "container_1352994193343_0001_02_000001"
, appAttemptToken2, clientTokenKey2, dispatcher);
ApplicationAttemptId attemptIdRemoved = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0002_000001"
);
ApplicationId appIdRemoved = attemptIdRemoved.GetApplicationId();
StoreApp(store, appIdRemoved, submitTime, startTime);
StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null
, null, dispatcher);
verifier.AfterStoreAppAttempt(store, attemptIdRemoved);
RMApp mockRemovedApp = Org.Mockito.Mockito.Mock <RMApp>();
RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics
>();
Dictionary <ApplicationAttemptId, RMAppAttempt> attempts = new Dictionary <ApplicationAttemptId
, RMAppAttempt>();
ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();
context.SetApplicationId(appIdRemoved);
Org.Mockito.Mockito.When(mockRemovedApp.GetSubmitTime()).ThenReturn(submitTime);
Org.Mockito.Mockito.When(mockRemovedApp.GetApplicationSubmissionContext()).ThenReturn
(context);
Org.Mockito.Mockito.When(mockRemovedApp.GetAppAttempts()).ThenReturn(attempts);
Org.Mockito.Mockito.When(mockRemovedApp.GetUser()).ThenReturn("user1");
RMAppAttempt mockRemovedAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(mockRemovedAttempt.GetAppAttemptId()).ThenReturn(attemptIdRemoved
);
Org.Mockito.Mockito.When(mockRemovedAttempt.GetRMAppAttemptMetrics()).ThenReturn(
mockRmAppAttemptMetrics);
Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()).
ThenReturn(new AggregateAppResourceUsage(0, 0));
attempts[attemptIdRemoved] = mockRemovedAttempt;
store.RemoveApplication(mockRemovedApp);
// remove application directory recursively.
StoreApp(store, appIdRemoved, submitTime, startTime);
StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null
, null, dispatcher);
store.RemoveApplication(mockRemovedApp);
// let things settle down
Sharpen.Thread.Sleep(1000);
store.Close();
// give tester a chance to modify app state in the store
ModifyAppState();
// load state
store = stateStoreHelper.GetRMStateStore();
store.SetRMDispatcher(dispatcher);
RMStateStore.RMState state = store.LoadState();
IDictionary <ApplicationId, ApplicationStateData> rmAppState = state.GetApplicationState
();
ApplicationStateData appState = rmAppState[appId1];
// app is loaded
NUnit.Framework.Assert.IsNotNull(appState);
// app is loaded correctly
NUnit.Framework.Assert.AreEqual(submitTime, appState.GetSubmitTime());
NUnit.Framework.Assert.AreEqual(startTime, appState.GetStartTime());
// submission context is loaded correctly
NUnit.Framework.Assert.AreEqual(appId1, appState.GetApplicationSubmissionContext(
).GetApplicationId());
ApplicationAttemptStateData attemptState = appState.GetAttempt(attemptId1);
// attempt1 is loaded correctly
NUnit.Framework.Assert.IsNotNull(attemptState);
NUnit.Framework.Assert.AreEqual(attemptId1, attemptState.GetAttemptId());
NUnit.Framework.Assert.AreEqual(-1000, attemptState.GetAMContainerExitStatus());
// attempt1 container is loaded correctly
NUnit.Framework.Assert.AreEqual(containerId1, attemptState.GetMasterContainer().GetId
());
// attempt1 client token master key is loaded correctly
Assert.AssertArrayEquals(clientTokenKey1.GetEncoded(), attemptState.GetAppAttemptTokens
().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
attemptState = appState.GetAttempt(attemptId2);
// attempt2 is loaded correctly
NUnit.Framework.Assert.IsNotNull(attemptState);
NUnit.Framework.Assert.AreEqual(attemptId2, attemptState.GetAttemptId());
// attempt2 container is loaded correctly
NUnit.Framework.Assert.AreEqual(containerId2, attemptState.GetMasterContainer().GetId
());
// attempt2 client token master key is loaded correctly
Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens
().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
//******* update application/attempt state *******//
ApplicationStateData appState2 = ApplicationStateData.NewInstance(appState.GetSubmitTime
(), appState.GetStartTime(), appState.GetUser(), appState.GetApplicationSubmissionContext
(), RMAppState.Finished, "appDiagnostics", 1234);
appState2.attempts.PutAll(appState.attempts);
store.UpdateApplicationState(appState2);
ApplicationAttemptStateData oldAttemptState = attemptState;
ApplicationAttemptStateData newAttemptState = ApplicationAttemptStateData.NewInstance
(oldAttemptState.GetAttemptId(), oldAttemptState.GetMasterContainer(), oldAttemptState
.GetAppAttemptTokens(), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished
, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.Succeeded, 100,
oldAttemptState.GetFinishTime(), 0, 0);
store.UpdateApplicationAttemptState(newAttemptState);
// test updating the state of an app/attempt whose initial state was not
// saved.
ApplicationId dummyAppId = ApplicationId.NewInstance(1234, 10);
ApplicationSubmissionContext dummyContext = new ApplicationSubmissionContextPBImpl
();
dummyContext.SetApplicationId(dummyAppId);
ApplicationStateData dummyApp = ApplicationStateData.NewInstance(appState.GetSubmitTime
(), appState.GetStartTime(), appState.GetUser(), dummyContext, RMAppState.Finished
, "appDiagnostics", 1234);
store.UpdateApplicationState(dummyApp);
ApplicationAttemptId dummyAttemptId = ApplicationAttemptId.NewInstance(dummyAppId
, 6);
ApplicationAttemptStateData dummyAttempt = ApplicationAttemptStateData.NewInstance
(dummyAttemptId, oldAttemptState.GetMasterContainer(), oldAttemptState.GetAppAttemptTokens
(), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished, "myTrackingUrl",
"attemptDiagnostics", FinalApplicationStatus.Succeeded, 111, oldAttemptState.GetFinishTime
(), 0, 0);
store.UpdateApplicationAttemptState(dummyAttempt);
// let things settle down
Sharpen.Thread.Sleep(1000);
store.Close();
// check updated application state.
store = stateStoreHelper.GetRMStateStore();
store.SetRMDispatcher(dispatcher);
RMStateStore.RMState newRMState = store.LoadState();
IDictionary <ApplicationId, ApplicationStateData> newRMAppState = newRMState.GetApplicationState
();
NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext
().GetApplicationId()]);
ApplicationStateData updatedAppState = newRMAppState[appId1];
NUnit.Framework.Assert.AreEqual(appState.GetApplicationSubmissionContext().GetApplicationId
(), updatedAppState.GetApplicationSubmissionContext().GetApplicationId());
NUnit.Framework.Assert.AreEqual(appState.GetSubmitTime(), updatedAppState.GetSubmitTime
());
NUnit.Framework.Assert.AreEqual(appState.GetStartTime(), updatedAppState.GetStartTime
());
NUnit.Framework.Assert.AreEqual(appState.GetUser(), updatedAppState.GetUser());
// new app state fields
NUnit.Framework.Assert.AreEqual(RMAppState.Finished, updatedAppState.GetState());
NUnit.Framework.Assert.AreEqual("appDiagnostics", updatedAppState.GetDiagnostics(
));
NUnit.Framework.Assert.AreEqual(1234, updatedAppState.GetFinishTime());
// check updated attempt state
NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext
().GetApplicationId()].GetAttempt(dummyAttemptId));
ApplicationAttemptStateData updatedAttemptState = updatedAppState.GetAttempt(newAttemptState
.GetAttemptId());
NUnit.Framework.Assert.AreEqual(oldAttemptState.GetAttemptId(), updatedAttemptState
.GetAttemptId());
NUnit.Framework.Assert.AreEqual(containerId2, updatedAttemptState.GetMasterContainer
().GetId());
Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens
().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
// new attempt state fields
NUnit.Framework.Assert.AreEqual(RMAppAttemptState.Finished, updatedAttemptState.GetState
());
NUnit.Framework.Assert.AreEqual("myTrackingUrl", updatedAttemptState.GetFinalTrackingUrl
());
NUnit.Framework.Assert.AreEqual("attemptDiagnostics", updatedAttemptState.GetDiagnostics
());
NUnit.Framework.Assert.AreEqual(100, updatedAttemptState.GetAMContainerExitStatus
());
NUnit.Framework.Assert.AreEqual(FinalApplicationStatus.Succeeded, updatedAttemptState
.GetFinalApplicationStatus());
// assert store is in expected state after everything is cleaned
NUnit.Framework.Assert.IsTrue(stateStoreHelper.IsFinalStateValid());
store.Close();
}
