/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestMRAppMasterMaxAppAttempts()
{
// No matter what's the maxAppAttempt or attempt id, the isLastRetry always
// equals to false
bool[] expectedBools = new bool[] { false, false, false };
string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
string containerIdStr = "container_1317529182569_0004_000002_1";
string userName = "******";
ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId
(applicationAttemptIdStr);
ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr);
JobConf conf = new JobConf();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
FilePath stagingDir = new FilePath(MRApps.GetStagingAreaDir(conf, userName).ToString
());
stagingDir.Mkdirs();
for (int i = 0; i < expectedBools.Length; ++i)
{
MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId
, "host", -1, -1, Runtime.CurrentTimeMillis(), false, true);
MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName);
NUnit.Framework.Assert.AreEqual("isLastAMRetry is correctly computed.", expectedBools
[i], appMaster.IsLastAMRetry());
}
}
/// <exception cref="System.IO.IOException"/>
public virtual IDictionary <ApplicationAttemptId, ApplicationAttemptHistoryData> GetApplicationAttempts
(ApplicationId appId)
{
IDictionary <ApplicationAttemptId, ApplicationAttemptHistoryData> historyDataMap =
new Dictionary <ApplicationAttemptId, ApplicationAttemptHistoryData>();
FileSystemApplicationHistoryStore.HistoryFileReader hfReader = GetHistoryFileReader
(appId);
try
{
while (hfReader.HasNext())
{
FileSystemApplicationHistoryStore.HistoryFileReader.Entry entry = hfReader.Next();
if (entry.key.id.StartsWith(ConverterUtils.ApplicationAttemptPrefix))
{
ApplicationAttemptId appAttemptId = ConverterUtils.ToApplicationAttemptId(entry.key
.id);
if (appAttemptId.GetApplicationId().Equals(appId))
{
ApplicationAttemptHistoryData historyData = historyDataMap[appAttemptId];
if (historyData == null)
{
historyData = ApplicationAttemptHistoryData.NewInstance(appAttemptId, null, -1, null
, null, null, FinalApplicationStatus.Undefined, null);
historyDataMap[appAttemptId] = historyData;
}
if (entry.key.suffix.Equals(StartDataSuffix))
{
MergeApplicationAttemptHistoryData(historyData, ParseApplicationAttemptStartData(
entry.value));
}
else
{
if (entry.key.suffix.Equals(FinishDataSuffix))
{
MergeApplicationAttemptHistoryData(historyData, ParseApplicationAttemptFinishData
(entry.value));
}
}
}
}
}
Log.Info("Completed reading history information of all application" + " attempts of application "
+ appId);
}
catch (IOException)
{
Log.Info("Error when reading history information of some application" + " attempts of application "
+ appId);
}
finally
{
hfReader.Close();
}
return(historyDataMap);
}
/// <summary>Prints the application attempt report for an application attempt id.</summary>
/// <param name="applicationAttemptId"/>
/// <returns>exitCode</returns>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private int PrintApplicationAttemptReport(string applicationAttemptId)
{
ApplicationAttemptReport appAttemptReport = null;
try
{
appAttemptReport = client.GetApplicationAttemptReport(ConverterUtils.ToApplicationAttemptId
(applicationAttemptId));
}
catch (ApplicationNotFoundException)
{
sysout.WriteLine("Application for AppAttempt with id '" + applicationAttemptId +
"' doesn't exist in RM or Timeline Server.");
return(-1);
}
catch (ApplicationAttemptNotFoundException)
{
sysout.WriteLine("Application Attempt with id '" + applicationAttemptId + "' doesn't exist in RM or Timeline Server."
);
return(-1);
}
// Use PrintWriter.println, which uses correct platform line ending.
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter appAttemptReportStr = new PrintWriter(new OutputStreamWriter(baos, Sharpen.Extensions.GetEncoding
("UTF-8")));
if (appAttemptReport != null)
{
appAttemptReportStr.WriteLine("Application Attempt Report : ");
appAttemptReportStr.Write("\tApplicationAttempt-Id : ");
appAttemptReportStr.WriteLine(appAttemptReport.GetApplicationAttemptId());
appAttemptReportStr.Write("\tState : ");
appAttemptReportStr.WriteLine(appAttemptReport.GetYarnApplicationAttemptState());
appAttemptReportStr.Write("\tAMContainer : ");
appAttemptReportStr.WriteLine(appAttemptReport.GetAMContainerId().ToString());
appAttemptReportStr.Write("\tTracking-URL : ");
appAttemptReportStr.WriteLine(appAttemptReport.GetTrackingUrl());
appAttemptReportStr.Write("\tRPC Port : ");
appAttemptReportStr.WriteLine(appAttemptReport.GetRpcPort());
appAttemptReportStr.Write("\tAM Host : ");
appAttemptReportStr.WriteLine(appAttemptReport.GetHost());
appAttemptReportStr.Write("\tDiagnostics : ");
appAttemptReportStr.Write(appAttemptReport.GetDiagnostics());
}
else
{
appAttemptReportStr.Write("Application Attempt with id '" + applicationAttemptId
+ "' doesn't exist in Timeline Server.");
appAttemptReportStr.Close();
sysout.WriteLine(baos.ToString("UTF-8"));
return(-1);
}
appAttemptReportStr.Close();
sysout.WriteLine(baos.ToString("UTF-8"));
return(0);
}
public virtual void TestBasic()
{
AppContext mockContext = Org.Mockito.Mockito.Mock <AppContext>();
OutputCommitter mockCommitter = Org.Mockito.Mockito.Mock <OutputCommitter>();
Clock mockClock = Org.Mockito.Mockito.Mock <Clock>();
CommitterEventHandler handler = new CommitterEventHandler(mockContext, mockCommitter
, new TestCommitterEventHandler.TestingRMHeartbeatHandler());
YarnConfiguration conf = new YarnConfiguration();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
JobContext mockJobContext = Org.Mockito.Mockito.Mock <JobContext>();
ApplicationAttemptId attemptid = ConverterUtils.ToApplicationAttemptId("appattempt_1234567890000_0001_0"
);
JobId jobId = TypeConverter.ToYarn(TypeConverter.FromYarn(attemptid.GetApplicationId
()));
TestCommitterEventHandler.WaitForItHandler waitForItHandler = new TestCommitterEventHandler.WaitForItHandler
();
Org.Mockito.Mockito.When(mockContext.GetApplicationID()).ThenReturn(attemptid.GetApplicationId
());
Org.Mockito.Mockito.When(mockContext.GetApplicationAttemptId()).ThenReturn(attemptid
);
Org.Mockito.Mockito.When(mockContext.GetEventHandler()).ThenReturn(waitForItHandler
);
Org.Mockito.Mockito.When(mockContext.GetClock()).ThenReturn(mockClock);
handler.Init(conf);
handler.Start();
try
{
handler.Handle(new CommitterJobCommitEvent(jobId, mockJobContext));
string user = UserGroupInformation.GetCurrentUser().GetShortUserName();
Path startCommitFile = MRApps.GetStartJobCommitFile(conf, user, jobId);
Path endCommitSuccessFile = MRApps.GetEndJobCommitSuccessFile(conf, user, jobId);
Path endCommitFailureFile = MRApps.GetEndJobCommitFailureFile(conf, user, jobId);
Org.Apache.Hadoop.Yarn.Event.Event e = waitForItHandler.GetAndClearEvent();
NUnit.Framework.Assert.IsNotNull(e);
NUnit.Framework.Assert.IsTrue(e is JobCommitCompletedEvent);
FileSystem fs = FileSystem.Get(conf);
NUnit.Framework.Assert.IsTrue(startCommitFile.ToString(), fs.Exists(startCommitFile
));
NUnit.Framework.Assert.IsTrue(endCommitSuccessFile.ToString(), fs.Exists(endCommitSuccessFile
));
NUnit.Framework.Assert.IsFalse(endCommitFailureFile.ToString(), fs.Exists(endCommitFailureFile
));
Org.Mockito.Mockito.Verify(mockCommitter).CommitJob(Matchers.Any <JobContext>());
}
finally
{
handler.Stop();
}
}
/// <exception cref="System.Exception"/>
protected internal override void ModifyAppState()
{
// imitate appAttemptFile1 is still .new, but old one is deleted
string appAttemptIdStr1 = "appattempt_1352994193343_0001_000001";
ApplicationAttemptId attemptId1 = ConverterUtils.ToApplicationAttemptId(appAttemptIdStr1
);
Path appDir = fsTester.store.GetAppDir(attemptId1.GetApplicationId().ToString());
Path appAttemptFile1 = new Path(appDir, attemptId1.ToString() + ".new");
FileSystemRMStateStore fileSystemRMStateStore = (FileSystemRMStateStore)fsTester.
GetRMStateStore();
fileSystemRMStateStore.RenameFile(appAttemptFile1, new Path(appAttemptFile1.GetParent
(), appAttemptFile1.GetName() + ".new"));
}
/// <exception cref="System.IO.IOException"/>
private ApplicationAttemptStateData CreateAttemptState(string itemName, byte[] data
)
{
ApplicationAttemptId attemptId = ConverterUtils.ToApplicationAttemptId(itemName);
ApplicationAttemptStateDataPBImpl attemptState = new ApplicationAttemptStateDataPBImpl
(YarnServerResourceManagerRecoveryProtos.ApplicationAttemptStateDataProto.ParseFrom
(data));
if (!attemptId.Equals(attemptState.GetAttemptId()))
{
throw new YarnRuntimeException("The database entry for " + attemptId + " contains data for "
+ attemptState.GetAttemptId());
}
return(attemptState);
}
protected internal static ApplicationAttemptId ParseApplicationAttemptId(string appAttemptId
)
{
if (appAttemptId == null || appAttemptId.IsEmpty())
{
throw new NotFoundException("appAttemptId, " + appAttemptId + ", is empty or null"
);
}
ApplicationAttemptId aaid = ConverterUtils.ToApplicationAttemptId(appAttemptId);
if (aaid == null)
{
throw new NotFoundException("appAttemptId is null");
}
return(aaid);
}
public virtual void TestMRAppMasterForDifferentUser()
{
string applicationAttemptIdStr = "appattempt_1317529182569_0004_000001";
string containerIdStr = "container_1317529182569_0004_000001_1";
string userName = "******";
ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId
(applicationAttemptIdStr);
ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr);
MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId
, "host", -1, -1, Runtime.CurrentTimeMillis());
JobConf conf = new JobConf();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName);
Path userPath = new Path(stagingDir, userName);
Path userStagingPath = new Path(userPath, ".staging");
NUnit.Framework.Assert.AreEqual(userStagingPath.ToString(), appMaster.stagingDirPath
.ToString());
}
/// <summary>Lists the containers matching the given application attempts</summary>
/// <param name="appAttemptId"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private void ListContainers(string appAttemptId)
{
PrintWriter writer = new PrintWriter(new OutputStreamWriter(sysout, Sharpen.Extensions.GetEncoding
("UTF-8")));
IList <ContainerReport> appsReport = client.GetContainers(ConverterUtils.ToApplicationAttemptId
(appAttemptId));
writer.WriteLine("Total number of containers " + ":" + appsReport.Count);
writer.Printf(ContainerPattern, "Container-Id", "Start Time", "Finish Time", "State"
, "Host", "Node Http Address", "LOG-URL");
foreach (ContainerReport containerReport in appsReport)
{
writer.Printf(ContainerPattern, containerReport.GetContainerId(), Times.Format(containerReport
.GetCreationTime()), Times.Format(containerReport.GetFinishTime()), containerReport
.GetContainerState(), containerReport.GetAssignedNode(), containerReport.GetNodeHttpAddress
() == null ? "N/A" : containerReport.GetNodeHttpAddress(), containerReport.GetLogUrl
());
}
writer.Flush();
}
public virtual void TestMRAppMasterSuccessLock()
{
string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
string containerIdStr = "container_1317529182569_0004_000002_1";
string userName = "******";
JobConf conf = new JobConf();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId
(applicationAttemptIdStr);
JobId jobId = TypeConverter.ToYarn(TypeConverter.FromYarn(applicationAttemptId.GetApplicationId
()));
Path start = MRApps.GetStartJobCommitFile(conf, userName, jobId);
Path end = MRApps.GetEndJobCommitSuccessFile(conf, userName, jobId);
FileSystem fs = FileSystem.Get(conf);
fs.Create(start).Close();
fs.Create(end).Close();
ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr);
MRAppMaster appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host"
, -1, -1, Runtime.CurrentTimeMillis(), false, false);
bool caught = false;
try
{
MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName);
}
catch (IOException e)
{
//The IO Exception is expected
Log.Info("Caught expected Exception", e);
caught = true;
}
NUnit.Framework.Assert.IsTrue(caught);
NUnit.Framework.Assert.IsTrue(appMaster.errorHappenedShutDown);
NUnit.Framework.Assert.AreEqual(JobStateInternal.Succeeded, appMaster.forcedState
);
appMaster.Stop();
// verify the final status is SUCCEEDED
VerifyFailedStatus((MRAppMasterTest)appMaster, "SUCCEEDED");
}
/// <exception cref="System.Exception"/>
public virtual void TestFSRMStateStore()
{
HdfsConfiguration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
try
{
fsTester = new TestFSRMStateStore.TestFSRMStateStoreTester(this, cluster, false);
// If the state store is FileSystemRMStateStore then add corrupted entry.
// It should discard the entry and remove it from file system.
FSDataOutputStream fsOut = null;
FileSystemRMStateStore fileSystemRMStateStore = (FileSystemRMStateStore)fsTester.
GetRMStateStore();
string appAttemptIdStr3 = "appattempt_1352994193343_0001_000003";
ApplicationAttemptId attemptId3 = ConverterUtils.ToApplicationAttemptId(appAttemptIdStr3
);
Path appDir = fsTester.store.GetAppDir(attemptId3.GetApplicationId().ToString());
Path tempAppAttemptFile = new Path(appDir, attemptId3.ToString() + ".tmp");
fsOut = fileSystemRMStateStore.fs.Create(tempAppAttemptFile, false);
fsOut.Write(Sharpen.Runtime.GetBytesForString("Some random data "));
fsOut.Close();
TestRMAppStateStore(fsTester);
NUnit.Framework.Assert.IsFalse(fsTester.workingDirPathURI.GetFileSystem(conf).Exists
(tempAppAttemptFile));
TestRMDTSecretManagerStateStore(fsTester);
TestCheckVersion(fsTester);
TestEpoch(fsTester);
TestAppDeletion(fsTester);
TestDeleteStore(fsTester);
TestAMRMTokenSecretManagerStateStore(fsTester);
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestMRAppMasterMissingStaging()
{
string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
string containerIdStr = "container_1317529182569_0004_000002_1";
string userName = "******";
JobConf conf = new JobConf();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId
(applicationAttemptIdStr);
//Delete the staging directory
FilePath dir = new FilePath(stagingDir);
if (dir.Exists())
{
FileUtils.DeleteDirectory(dir);
}
ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr);
MRAppMaster appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host"
, -1, -1, Runtime.CurrentTimeMillis(), false, false);
bool caught = false;
try
{
MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName);
}
catch (IOException e)
{
//The IO Exception is expected
Log.Info("Caught expected Exception", e);
caught = true;
}
NUnit.Framework.Assert.IsTrue(caught);
NUnit.Framework.Assert.IsTrue(appMaster.errorHappenedShutDown);
//Copying the history file is disabled, but it is not really visible from
//here
NUnit.Framework.Assert.AreEqual(JobStateInternal.Error, appMaster.forcedState);
appMaster.Stop();
}
public virtual void TestDuplicateRMAppDeletion()
{
TestZKRMStateStore.TestZKRMStateStoreTester zkTester = new TestZKRMStateStore.TestZKRMStateStoreTester
(this);
long submitTime = Runtime.CurrentTimeMillis();
long startTime = Runtime.CurrentTimeMillis() + 1234;
RMStateStore store = zkTester.GetRMStateStore();
RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
();
store.SetRMDispatcher(dispatcher);
ApplicationAttemptId attemptIdRemoved = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0002_000001"
);
ApplicationId appIdRemoved = attemptIdRemoved.GetApplicationId();
StoreApp(store, appIdRemoved, submitTime, startTime);
StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null
, null, dispatcher);
ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();
context.SetApplicationId(appIdRemoved);
ApplicationStateData appStateRemoved = ApplicationStateData.NewInstance(submitTime
, startTime, context, "user1");
appStateRemoved.attempts[attemptIdRemoved] = null;
store.RemoveApplicationStateInternal(appStateRemoved);
try
{
store.RemoveApplicationStateInternal(appStateRemoved);
}
catch (KeeperException.NoNodeException)
{
NUnit.Framework.Assert.Fail("NoNodeException should not happen.");
}
store.Close();
}
private static ApplicationHistoryManagerOnTimelineStore.ApplicationReportExt ConvertToApplicationReport
(TimelineEntity entity, ApplicationHistoryManagerOnTimelineStore.ApplicationReportField
field)
{
string user = null;
string queue = null;
string name = null;
string type = null;
long createdTime = 0;
long finishedTime = 0;
ApplicationAttemptId latestApplicationAttemptId = null;
string diagnosticsInfo = null;
FinalApplicationStatus finalStatus = FinalApplicationStatus.Undefined;
YarnApplicationState state = null;
ApplicationResourceUsageReport appResources = null;
IDictionary <ApplicationAccessType, string> appViewACLs = new Dictionary <ApplicationAccessType
, string>();
IDictionary <string, object> entityInfo = entity.GetOtherInfo();
if (entityInfo != null)
{
if (entityInfo.Contains(ApplicationMetricsConstants.UserEntityInfo))
{
user = entityInfo[ApplicationMetricsConstants.UserEntityInfo].ToString();
}
if (entityInfo.Contains(ApplicationMetricsConstants.AppViewAclsEntityInfo))
{
string appViewACLsStr = entityInfo[ApplicationMetricsConstants.AppViewAclsEntityInfo
].ToString();
if (appViewACLsStr.Length > 0)
{
appViewACLs[ApplicationAccessType.ViewApp] = appViewACLsStr;
}
}
if (field == ApplicationHistoryManagerOnTimelineStore.ApplicationReportField.UserAndAcls)
{
return(new ApplicationHistoryManagerOnTimelineStore.ApplicationReportExt(ApplicationReport
.NewInstance(ConverterUtils.ToApplicationId(entity.GetEntityId()), latestApplicationAttemptId
, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime,
finishedTime, finalStatus, null, null, 1.0F, type, null), appViewACLs));
}
if (entityInfo.Contains(ApplicationMetricsConstants.QueueEntityInfo))
{
queue = entityInfo[ApplicationMetricsConstants.QueueEntityInfo].ToString();
}
if (entityInfo.Contains(ApplicationMetricsConstants.NameEntityInfo))
{
name = entityInfo[ApplicationMetricsConstants.NameEntityInfo].ToString();
}
if (entityInfo.Contains(ApplicationMetricsConstants.TypeEntityInfo))
{
type = entityInfo[ApplicationMetricsConstants.TypeEntityInfo].ToString();
}
if (entityInfo.Contains(ApplicationMetricsConstants.AppCpuMetrics))
{
long vcoreSeconds = long.Parse(entityInfo[ApplicationMetricsConstants.AppCpuMetrics
].ToString());
long memorySeconds = long.Parse(entityInfo[ApplicationMetricsConstants.AppMemMetrics
].ToString());
appResources = ApplicationResourceUsageReport.NewInstance(0, 0, null, null, null,
memorySeconds, vcoreSeconds);
}
}
IList <TimelineEvent> events = entity.GetEvents();
if (events != null)
{
foreach (TimelineEvent @event in events)
{
if (@event.GetEventType().Equals(ApplicationMetricsConstants.CreatedEventType))
{
createdTime = @event.GetTimestamp();
}
else
{
if (@event.GetEventType().Equals(ApplicationMetricsConstants.FinishedEventType))
{
finishedTime = @event.GetTimestamp();
IDictionary <string, object> eventInfo = @event.GetEventInfo();
if (eventInfo == null)
{
continue;
}
if (eventInfo.Contains(ApplicationMetricsConstants.LatestAppAttemptEventInfo))
{
latestApplicationAttemptId = ConverterUtils.ToApplicationAttemptId(eventInfo[ApplicationMetricsConstants
.LatestAppAttemptEventInfo].ToString());
}
if (eventInfo.Contains(ApplicationMetricsConstants.DiagnosticsInfoEventInfo))
{
diagnosticsInfo = eventInfo[ApplicationMetricsConstants.DiagnosticsInfoEventInfo]
.ToString();
}
if (eventInfo.Contains(ApplicationMetricsConstants.FinalStatusEventInfo))
{
finalStatus = FinalApplicationStatus.ValueOf(eventInfo[ApplicationMetricsConstants
.FinalStatusEventInfo].ToString());
}
if (eventInfo.Contains(ApplicationMetricsConstants.StateEventInfo))
{
state = YarnApplicationState.ValueOf(eventInfo[ApplicationMetricsConstants.StateEventInfo
].ToString());
}
}
}
}
}
return(new ApplicationHistoryManagerOnTimelineStore.ApplicationReportExt(ApplicationReport
.NewInstance(ConverterUtils.ToApplicationId(entity.GetEntityId()), latestApplicationAttemptId
, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime,
finishedTime, finalStatus, appResources, null, 1.0F, type, null), appViewACLs));
}
/// <returns>the ApplicationAttemptId</returns>
public virtual ApplicationAttemptId GetAppAttemptId()
{
return(ConverterUtils.ToApplicationAttemptId(datum.applicationAttemptId.ToString(
)));
}
/// <exception cref="System.Exception"/>
internal virtual void TestRMAppStateStore(RMStateStoreTestBase.RMStateStoreHelper
stateStoreHelper, RMStateStoreTestBase.StoreStateVerifier verifier)
{
long submitTime = Runtime.CurrentTimeMillis();
long startTime = Runtime.CurrentTimeMillis() + 1234;
Configuration conf = new YarnConfiguration();
RMStateStore store = stateStoreHelper.GetRMStateStore();
RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
();
store.SetRMDispatcher(dispatcher);
RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();
Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store);
AMRMTokenSecretManager appTokenMgr = Org.Mockito.Mockito.Spy(new AMRMTokenSecretManager
(conf, rmContext));
MasterKeyData masterKeyData = appTokenMgr.CreateNewMasterKey();
Org.Mockito.Mockito.When(appTokenMgr.GetMasterKey()).ThenReturn(masterKeyData);
ClientToAMTokenSecretManagerInRM clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM
();
ApplicationAttemptId attemptId1 = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0001_000001"
);
ApplicationId appId1 = attemptId1.GetApplicationId();
StoreApp(store, appId1, submitTime, startTime);
verifier.AfterStoreApp(store, appId1);
// create application token and client token key for attempt1
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken1 = GenerateAMRMToken
(attemptId1, appTokenMgr);
SecretKey clientTokenKey1 = clientToAMTokenMgr.CreateMasterKey(attemptId1);
ContainerId containerId1 = StoreAttempt(store, attemptId1, "container_1352994193343_0001_01_000001"
, appAttemptToken1, clientTokenKey1, dispatcher);
string appAttemptIdStr2 = "appattempt_1352994193343_0001_000002";
ApplicationAttemptId attemptId2 = ConverterUtils.ToApplicationAttemptId(appAttemptIdStr2
);
// create application token and client token key for attempt2
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken2 = GenerateAMRMToken
(attemptId2, appTokenMgr);
SecretKey clientTokenKey2 = clientToAMTokenMgr.CreateMasterKey(attemptId2);
ContainerId containerId2 = StoreAttempt(store, attemptId2, "container_1352994193343_0001_02_000001"
, appAttemptToken2, clientTokenKey2, dispatcher);
ApplicationAttemptId attemptIdRemoved = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0002_000001"
);
ApplicationId appIdRemoved = attemptIdRemoved.GetApplicationId();
StoreApp(store, appIdRemoved, submitTime, startTime);
StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null
, null, dispatcher);
verifier.AfterStoreAppAttempt(store, attemptIdRemoved);
RMApp mockRemovedApp = Org.Mockito.Mockito.Mock <RMApp>();
RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics
>();
Dictionary <ApplicationAttemptId, RMAppAttempt> attempts = new Dictionary <ApplicationAttemptId
, RMAppAttempt>();
ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();
context.SetApplicationId(appIdRemoved);
Org.Mockito.Mockito.When(mockRemovedApp.GetSubmitTime()).ThenReturn(submitTime);
Org.Mockito.Mockito.When(mockRemovedApp.GetApplicationSubmissionContext()).ThenReturn
(context);
Org.Mockito.Mockito.When(mockRemovedApp.GetAppAttempts()).ThenReturn(attempts);
Org.Mockito.Mockito.When(mockRemovedApp.GetUser()).ThenReturn("user1");
RMAppAttempt mockRemovedAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(mockRemovedAttempt.GetAppAttemptId()).ThenReturn(attemptIdRemoved
);
Org.Mockito.Mockito.When(mockRemovedAttempt.GetRMAppAttemptMetrics()).ThenReturn(
mockRmAppAttemptMetrics);
Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()).
ThenReturn(new AggregateAppResourceUsage(0, 0));
attempts[attemptIdRemoved] = mockRemovedAttempt;
store.RemoveApplication(mockRemovedApp);
// remove application directory recursively.
StoreApp(store, appIdRemoved, submitTime, startTime);
StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null
, null, dispatcher);
store.RemoveApplication(mockRemovedApp);
// let things settle down
Sharpen.Thread.Sleep(1000);
store.Close();
// give tester a chance to modify app state in the store
ModifyAppState();
// load state
store = stateStoreHelper.GetRMStateStore();
store.SetRMDispatcher(dispatcher);
RMStateStore.RMState state = store.LoadState();
IDictionary <ApplicationId, ApplicationStateData> rmAppState = state.GetApplicationState
();
ApplicationStateData appState = rmAppState[appId1];
// app is loaded
NUnit.Framework.Assert.IsNotNull(appState);
// app is loaded correctly
NUnit.Framework.Assert.AreEqual(submitTime, appState.GetSubmitTime());
NUnit.Framework.Assert.AreEqual(startTime, appState.GetStartTime());
// submission context is loaded correctly
NUnit.Framework.Assert.AreEqual(appId1, appState.GetApplicationSubmissionContext(
).GetApplicationId());
ApplicationAttemptStateData attemptState = appState.GetAttempt(attemptId1);
// attempt1 is loaded correctly
NUnit.Framework.Assert.IsNotNull(attemptState);
NUnit.Framework.Assert.AreEqual(attemptId1, attemptState.GetAttemptId());
NUnit.Framework.Assert.AreEqual(-1000, attemptState.GetAMContainerExitStatus());
// attempt1 container is loaded correctly
NUnit.Framework.Assert.AreEqual(containerId1, attemptState.GetMasterContainer().GetId
());
// attempt1 client token master key is loaded correctly
Assert.AssertArrayEquals(clientTokenKey1.GetEncoded(), attemptState.GetAppAttemptTokens
().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
attemptState = appState.GetAttempt(attemptId2);
// attempt2 is loaded correctly
NUnit.Framework.Assert.IsNotNull(attemptState);
NUnit.Framework.Assert.AreEqual(attemptId2, attemptState.GetAttemptId());
// attempt2 container is loaded correctly
NUnit.Framework.Assert.AreEqual(containerId2, attemptState.GetMasterContainer().GetId
());
// attempt2 client token master key is loaded correctly
Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens
().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
//******* update application/attempt state *******//
ApplicationStateData appState2 = ApplicationStateData.NewInstance(appState.GetSubmitTime
(), appState.GetStartTime(), appState.GetUser(), appState.GetApplicationSubmissionContext
(), RMAppState.Finished, "appDiagnostics", 1234);
appState2.attempts.PutAll(appState.attempts);
store.UpdateApplicationState(appState2);
ApplicationAttemptStateData oldAttemptState = attemptState;
ApplicationAttemptStateData newAttemptState = ApplicationAttemptStateData.NewInstance
(oldAttemptState.GetAttemptId(), oldAttemptState.GetMasterContainer(), oldAttemptState
.GetAppAttemptTokens(), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished
, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.Succeeded, 100,
oldAttemptState.GetFinishTime(), 0, 0);
store.UpdateApplicationAttemptState(newAttemptState);
// test updating the state of an app/attempt whose initial state was not
// saved.
ApplicationId dummyAppId = ApplicationId.NewInstance(1234, 10);
ApplicationSubmissionContext dummyContext = new ApplicationSubmissionContextPBImpl
();
dummyContext.SetApplicationId(dummyAppId);
ApplicationStateData dummyApp = ApplicationStateData.NewInstance(appState.GetSubmitTime
(), appState.GetStartTime(), appState.GetUser(), dummyContext, RMAppState.Finished
, "appDiagnostics", 1234);
store.UpdateApplicationState(dummyApp);
ApplicationAttemptId dummyAttemptId = ApplicationAttemptId.NewInstance(dummyAppId
, 6);
ApplicationAttemptStateData dummyAttempt = ApplicationAttemptStateData.NewInstance
(dummyAttemptId, oldAttemptState.GetMasterContainer(), oldAttemptState.GetAppAttemptTokens
(), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished, "myTrackingUrl",
"attemptDiagnostics", FinalApplicationStatus.Succeeded, 111, oldAttemptState.GetFinishTime
(), 0, 0);
store.UpdateApplicationAttemptState(dummyAttempt);
// let things settle down
Sharpen.Thread.Sleep(1000);
store.Close();
// check updated application state.
store = stateStoreHelper.GetRMStateStore();
store.SetRMDispatcher(dispatcher);
RMStateStore.RMState newRMState = store.LoadState();
IDictionary <ApplicationId, ApplicationStateData> newRMAppState = newRMState.GetApplicationState
();
NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext
().GetApplicationId()]);
ApplicationStateData updatedAppState = newRMAppState[appId1];
NUnit.Framework.Assert.AreEqual(appState.GetApplicationSubmissionContext().GetApplicationId
(), updatedAppState.GetApplicationSubmissionContext().GetApplicationId());
NUnit.Framework.Assert.AreEqual(appState.GetSubmitTime(), updatedAppState.GetSubmitTime
());
NUnit.Framework.Assert.AreEqual(appState.GetStartTime(), updatedAppState.GetStartTime
());
NUnit.Framework.Assert.AreEqual(appState.GetUser(), updatedAppState.GetUser());
// new app state fields
NUnit.Framework.Assert.AreEqual(RMAppState.Finished, updatedAppState.GetState());
NUnit.Framework.Assert.AreEqual("appDiagnostics", updatedAppState.GetDiagnostics(
));
NUnit.Framework.Assert.AreEqual(1234, updatedAppState.GetFinishTime());
// check updated attempt state
NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext
().GetApplicationId()].GetAttempt(dummyAttemptId));
ApplicationAttemptStateData updatedAttemptState = updatedAppState.GetAttempt(newAttemptState
.GetAttemptId());
NUnit.Framework.Assert.AreEqual(oldAttemptState.GetAttemptId(), updatedAttemptState
.GetAttemptId());
NUnit.Framework.Assert.AreEqual(containerId2, updatedAttemptState.GetMasterContainer
().GetId());
Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens
().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
// new attempt state fields
NUnit.Framework.Assert.AreEqual(RMAppAttemptState.Finished, updatedAttemptState.GetState
());
NUnit.Framework.Assert.AreEqual("myTrackingUrl", updatedAttemptState.GetFinalTrackingUrl
());
NUnit.Framework.Assert.AreEqual("attemptDiagnostics", updatedAttemptState.GetDiagnostics
());
NUnit.Framework.Assert.AreEqual(100, updatedAttemptState.GetAMContainerExitStatus
());
NUnit.Framework.Assert.AreEqual(FinalApplicationStatus.Succeeded, updatedAttemptState
.GetFinalApplicationStatus());
// assert store is in expected state after everything is cleaned
NUnit.Framework.Assert.IsTrue(stateStoreHelper.IsFinalStateValid());
store.Close();
}
/// <exception cref="System.IO.IOException"/>
public override NMStateStoreService.RecoveredNMTokensState LoadNMTokensState()
{
NMStateStoreService.RecoveredNMTokensState state = new NMStateStoreService.RecoveredNMTokensState
();
state.applicationMasterKeys = new Dictionary <ApplicationAttemptId, MasterKey>();
LeveldbIterator iter = null;
try
{
iter = new LeveldbIterator(db);
iter.Seek(JniDBFactory.Bytes(NmTokensKeyPrefix));
while (iter.HasNext())
{
KeyValuePair <byte[], byte[]> entry = iter.Next();
string fullKey = JniDBFactory.AsString(entry.Key);
if (!fullKey.StartsWith(NmTokensKeyPrefix))
{
break;
}
string key = Sharpen.Runtime.Substring(fullKey, NmTokensKeyPrefix.Length);
if (key.Equals(CurrentMasterKeySuffix))
{
state.currentMasterKey = ParseMasterKey(entry.Value);
}
else
{
if (key.Equals(PrevMasterKeySuffix))
{
state.previousMasterKey = ParseMasterKey(entry.Value);
}
else
{
if (key.StartsWith(ApplicationAttemptId.appAttemptIdStrPrefix))
{
ApplicationAttemptId attempt;
try
{
attempt = ConverterUtils.ToApplicationAttemptId(key);
}
catch (ArgumentException e)
{
throw new IOException("Bad application master key state for " + fullKey, e);
}
state.applicationMasterKeys[attempt] = ParseMasterKey(entry.Value);
}
}
}
}
}
catch (DBException e)
{
throw new IOException(e);
}
finally
{
if (iter != null)
{
iter.Close();
}
}
return(state);
}
protected override void RenderData(HtmlBlock.Block html)
{
Hamlet.TBODY <Hamlet.TABLE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > tbody = html
.Table("#apps").Thead().Tr().Th(".id", "ID").Th(".user", "User").Th(".name", "Name"
).Th(".type", "Application Type").Th(".queue", "Queue").Th(".starttime", "StartTime"
).Th(".finishtime", "FinishTime").Th(".state", "State").Th(".finalstatus", "FinalStatus"
).Th(".progress", "Progress").Th(".ui", "Tracking UI").Th(".blacklisted", "Blacklisted Nodes"
).().().Tbody();
StringBuilder appsTableData = new StringBuilder("[\n");
foreach (ApplicationReport appReport in appReports)
{
// TODO: remove the following condition. It is still here because
// the history side implementation of ApplicationBaseProtocol
// hasn't filtering capability (YARN-1819).
if (!reqAppStates.IsEmpty() && !reqAppStates.Contains(appReport.GetYarnApplicationState
()))
{
continue;
}
AppInfo app = new AppInfo(appReport);
string blacklistedNodesCount = "N/A";
ICollection <string> nodes = RMAppAttemptBlock.GetBlacklistedNodes(rm, ConverterUtils
.ToApplicationAttemptId(app.GetCurrentAppAttemptId()));
if (nodes != null)
{
blacklistedNodesCount = nodes.Count.ToString();
}
string percent = string.Format("%.1f", app.GetProgress());
// AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js
appsTableData.Append("[\"<a href='").Append(Url("app", app.GetAppId())).Append("'>"
).Append(app.GetAppId()).Append("</a>\",\"").Append(StringEscapeUtils.EscapeJavaScript
(StringEscapeUtils.EscapeHtml(app.GetUser()))).Append("\",\"").Append(StringEscapeUtils
.EscapeJavaScript(StringEscapeUtils.EscapeHtml(app.GetName()))).Append("\",\"").
Append(StringEscapeUtils.EscapeJavaScript(StringEscapeUtils.EscapeHtml(app.GetType
()))).Append("\",\"").Append(StringEscapeUtils.EscapeJavaScript(StringEscapeUtils
.EscapeHtml(app.GetQueue()))).Append("\",\"").Append(app.GetStartedTime()).Append
("\",\"").Append(app.GetFinishedTime()).Append("\",\"").Append(app.GetAppState()
== null ? Unavailable : app.GetAppState()).Append("\",\"").Append(app.GetFinalAppStatus
()).Append("\",\"").Append("<br title='").Append(percent).Append("'> <div class='"
).Append(JQueryUI.CProgressbar).Append("' title='").Append(StringHelper.Join(percent
, '%')).Append("'> ").Append("<div class='").Append(JQueryUI.CProgressbarValue).
Append("' style='").Append(StringHelper.Join("width:", percent, '%')).Append("'> </div> </div>"
).Append("\",\"<a ");
// Progress bar
string trackingURL = app.GetTrackingUrl() == null || app.GetTrackingUrl().Equals(
Unavailable) ? null : app.GetTrackingUrl();
string trackingUI = app.GetTrackingUrl() == null || app.GetTrackingUrl().Equals(Unavailable
) ? "Unassigned" : app.GetAppState() == YarnApplicationState.Finished || app.GetAppState
() == YarnApplicationState.Failed || app.GetAppState() == YarnApplicationState.Killed
? "History" : "ApplicationMaster";
appsTableData.Append(trackingURL == null ? "#" : "href='" + trackingURL).Append("'>"
).Append(trackingUI).Append("</a>\",").Append("\"").Append(blacklistedNodesCount
).Append("\"],\n");
}
if (appsTableData[appsTableData.Length - 2] == ',')
{
appsTableData.Delete(appsTableData.Length - 2, appsTableData.Length - 1);
}
appsTableData.Append("]");
html.Script().$type("text/javascript").("var appsTableData=" + appsTableData).();
tbody.().();
}
private static ApplicationAttemptReport ConvertToApplicationAttemptReport(TimelineEntity
entity)
{
string host = null;
int rpcPort = -1;
ContainerId amContainerId = null;
string trackingUrl = null;
string originalTrackingUrl = null;
string diagnosticsInfo = null;
YarnApplicationAttemptState state = null;
IList <TimelineEvent> events = entity.GetEvents();
if (events != null)
{
foreach (TimelineEvent @event in events)
{
if (@event.GetEventType().Equals(AppAttemptMetricsConstants.RegisteredEventType))
{
IDictionary <string, object> eventInfo = @event.GetEventInfo();
if (eventInfo == null)
{
continue;
}
if (eventInfo.Contains(AppAttemptMetricsConstants.HostEventInfo))
{
host = eventInfo[AppAttemptMetricsConstants.HostEventInfo].ToString();
}
if (eventInfo.Contains(AppAttemptMetricsConstants.RpcPortEventInfo))
{
rpcPort = (int)eventInfo[AppAttemptMetricsConstants.RpcPortEventInfo];
}
if (eventInfo.Contains(AppAttemptMetricsConstants.MasterContainerEventInfo))
{
amContainerId = ConverterUtils.ToContainerId(eventInfo[AppAttemptMetricsConstants
.MasterContainerEventInfo].ToString());
}
}
else
{
if (@event.GetEventType().Equals(AppAttemptMetricsConstants.FinishedEventType))
{
IDictionary <string, object> eventInfo = @event.GetEventInfo();
if (eventInfo == null)
{
continue;
}
if (eventInfo.Contains(AppAttemptMetricsConstants.TrackingUrlEventInfo))
{
trackingUrl = eventInfo[AppAttemptMetricsConstants.TrackingUrlEventInfo].ToString
();
}
if (eventInfo.Contains(AppAttemptMetricsConstants.OriginalTrackingUrlEventInfo))
{
originalTrackingUrl = eventInfo[AppAttemptMetricsConstants.OriginalTrackingUrlEventInfo
].ToString();
}
if (eventInfo.Contains(AppAttemptMetricsConstants.DiagnosticsInfoEventInfo))
{
diagnosticsInfo = eventInfo[AppAttemptMetricsConstants.DiagnosticsInfoEventInfo].
ToString();
}
if (eventInfo.Contains(AppAttemptMetricsConstants.StateEventInfo))
{
state = YarnApplicationAttemptState.ValueOf(eventInfo[AppAttemptMetricsConstants.
StateEventInfo].ToString());
}
}
}
}
}
return(ApplicationAttemptReport.NewInstance(ConverterUtils.ToApplicationAttemptId
(entity.GetEntityId()), host, rpcPort, trackingUrl, originalTrackingUrl, diagnosticsInfo
, state, amContainerId));
}
public virtual void TestCommitWindow()
{
Configuration conf = new Configuration();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
AsyncDispatcher dispatcher = new AsyncDispatcher();
dispatcher.Init(conf);
dispatcher.Start();
TestCommitterEventHandler.TestingJobEventHandler jeh = new TestCommitterEventHandler.TestingJobEventHandler
();
dispatcher.Register(typeof(JobEventType), jeh);
SystemClock clock = new SystemClock();
AppContext appContext = Org.Mockito.Mockito.Mock <AppContext>();
ApplicationAttemptId attemptid = ConverterUtils.ToApplicationAttemptId("appattempt_1234567890000_0001_0"
);
Org.Mockito.Mockito.When(appContext.GetApplicationID()).ThenReturn(attemptid.GetApplicationId
());
Org.Mockito.Mockito.When(appContext.GetApplicationAttemptId()).ThenReturn(attemptid
);
Org.Mockito.Mockito.When(appContext.GetEventHandler()).ThenReturn(dispatcher.GetEventHandler
());
Org.Mockito.Mockito.When(appContext.GetClock()).ThenReturn(clock);
OutputCommitter committer = Org.Mockito.Mockito.Mock <OutputCommitter>();
TestCommitterEventHandler.TestingRMHeartbeatHandler rmhh = new TestCommitterEventHandler.TestingRMHeartbeatHandler
();
CommitterEventHandler ceh = new CommitterEventHandler(appContext, committer, rmhh
);
ceh.Init(conf);
ceh.Start();
// verify trying to commit when RM heartbeats are stale does not commit
ceh.Handle(new CommitterJobCommitEvent(null, null));
long timeToWaitMs = 5000;
while (rmhh.GetNumCallbacks() != 1 && timeToWaitMs > 0)
{
Sharpen.Thread.Sleep(10);
timeToWaitMs -= 10;
}
NUnit.Framework.Assert.AreEqual("committer did not register a heartbeat callback"
, 1, rmhh.GetNumCallbacks());
Org.Mockito.Mockito.Verify(committer, Org.Mockito.Mockito.Never()).CommitJob(Matchers.Any
<JobContext>());
NUnit.Framework.Assert.AreEqual("committer should not have committed", 0, jeh.numCommitCompletedEvents
);
// set a fresh heartbeat and verify commit completes
rmhh.SetLastHeartbeatTime(clock.GetTime());
timeToWaitMs = 5000;
while (jeh.numCommitCompletedEvents != 1 && timeToWaitMs > 0)
{
Sharpen.Thread.Sleep(10);
timeToWaitMs -= 10;
}
NUnit.Framework.Assert.AreEqual("committer did not complete commit after RM hearbeat"
, 1, jeh.numCommitCompletedEvents);
Org.Mockito.Mockito.Verify(committer, Org.Mockito.Mockito.Times(1)).CommitJob(Matchers.Any
<JobContext>());
//Clean up so we can try to commit again (Don't do this at home)
Cleanup();
// try to commit again and verify it goes through since the heartbeat
// is still fresh
ceh.Handle(new CommitterJobCommitEvent(null, null));
timeToWaitMs = 5000;
while (jeh.numCommitCompletedEvents != 2 && timeToWaitMs > 0)
{
Sharpen.Thread.Sleep(10);
timeToWaitMs -= 10;
}
NUnit.Framework.Assert.AreEqual("committer did not commit", 2, jeh.numCommitCompletedEvents
);
Org.Mockito.Mockito.Verify(committer, Org.Mockito.Mockito.Times(2)).CommitJob(Matchers.Any
<JobContext>());
ceh.Stop();
dispatcher.Stop();
}
protected override void Render(HtmlBlock.Block html)
{
string attemptid = $(YarnWebParams.ApplicationAttemptId);
if (attemptid.IsEmpty())
{
Puts("Bad request: requires application attempt ID");
return;
}
try
{
appAttemptId = ConverterUtils.ToApplicationAttemptId(attemptid);
}
catch (ArgumentException)
{
Puts("Invalid application attempt ID: " + attemptid);
return;
}
UserGroupInformation callerUGI = GetCallerUGI();
ApplicationAttemptReport appAttemptReport;
try
{
GetApplicationAttemptReportRequest request = GetApplicationAttemptReportRequest.NewInstance
(appAttemptId);
if (callerUGI == null)
{
appAttemptReport = appBaseProt.GetApplicationAttemptReport(request).GetApplicationAttemptReport
();
}
else
{
appAttemptReport = callerUGI.DoAs(new _PrivilegedExceptionAction_85(this, request
));
}
}
catch (Exception e)
{
string message = "Failed to read the application attempt " + appAttemptId + ".";
Log.Error(message, e);
html.P().(message).();
return;
}
if (appAttemptReport == null)
{
Puts("Application Attempt not found: " + attemptid);
return;
}
bool exceptionWhenGetContainerReports = false;
ICollection <ContainerReport> containers = null;
try
{
GetContainersRequest request = GetContainersRequest.NewInstance(appAttemptId);
if (callerUGI == null)
{
containers = appBaseProt.GetContainers(request).GetContainerList();
}
else
{
containers = callerUGI.DoAs(new _PrivilegedExceptionAction_115(this, request));
}
}
catch (RuntimeException)
{
// have this block to suppress the findbugs warning
exceptionWhenGetContainerReports = true;
}
catch (Exception)
{
exceptionWhenGetContainerReports = true;
}
AppAttemptInfo appAttempt = new AppAttemptInfo(appAttemptReport);
SetTitle(StringHelper.Join("Application Attempt ", attemptid));
string node = "N/A";
if (appAttempt.GetHost() != null && appAttempt.GetRpcPort() >= 0 && appAttempt.GetRpcPort
() < 65536)
{
node = appAttempt.GetHost() + ":" + appAttempt.GetRpcPort();
}
GenerateOverview(appAttemptReport, containers, appAttempt, node);
if (exceptionWhenGetContainerReports)
{
html.P().("Sorry, Failed to get containers for application attempt" + attemptid +
".").();
return;
}
CreateAttemptHeadRoomTable(html);
html.(typeof(InfoBlock));
CreateTablesForAttemptMetrics(html);
// Container Table
Hamlet.TBODY <Hamlet.TABLE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > tbody = html
.Table("#containers").Thead().Tr().Th(".id", "Container ID").Th(".node", "Node")
.Th(".exitstatus", "Container Exit Status").Th(".logs", "Logs").().().Tbody();
StringBuilder containersTableData = new StringBuilder("[\n");
foreach (ContainerReport containerReport in containers)
{
ContainerInfo container = new ContainerInfo(containerReport);
containersTableData.Append("[\"<a href='").Append(Url("container", container.GetContainerId
())).Append("'>").Append(container.GetContainerId()).Append("</a>\",\"<a ").Append
(container.GetNodeHttpAddress() == null ? "#" : "href='" + container.GetNodeHttpAddress
()).Append("'>").Append(container.GetNodeHttpAddress() == null ? "N/A" : StringEscapeUtils
.EscapeJavaScript(StringEscapeUtils.EscapeHtml(container.GetNodeHttpAddress())))
.Append("</a>\",\"").Append(container.GetContainerExitStatus()).Append("\",\"<a href='"
).Append(container.GetLogUrl() == null ? "#" : container.GetLogUrl()).Append("'>"
).Append(container.GetLogUrl() == null ? "N/A" : "Logs").Append("</a>\"],\n");
}
if (containersTableData[containersTableData.Length - 2] == ',')
{
containersTableData.Delete(containersTableData.Length - 2, containersTableData.Length
- 1);
}
containersTableData.Append("]");
html.Script().$type("text/javascript").("var containersTableData=" + containersTableData
).();
tbody.().();
}
public virtual void TestFencedState()
{
TestZKRMStateStore.TestZKRMStateStoreTester zkTester = new TestZKRMStateStore.TestZKRMStateStoreTester
(this);
RMStateStore store = zkTester.GetRMStateStore();
// Move state to FENCED from ACTIVE
store.UpdateFencedState();
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
long submitTime = Runtime.CurrentTimeMillis();
long startTime = submitTime + 1000;
// Add a new app
RMApp mockApp = Org.Mockito.Mockito.Mock <RMApp>();
ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();
Org.Mockito.Mockito.When(mockApp.GetSubmitTime()).ThenReturn(submitTime);
Org.Mockito.Mockito.When(mockApp.GetStartTime()).ThenReturn(startTime);
Org.Mockito.Mockito.When(mockApp.GetApplicationSubmissionContext()).ThenReturn(context
);
Org.Mockito.Mockito.When(mockApp.GetUser()).ThenReturn("test");
store.StoreNewApplication(mockApp);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// Add a new attempt
ClientToAMTokenSecretManagerInRM clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM
();
ApplicationAttemptId attemptId = ConverterUtils.ToApplicationAttemptId("appattempt_1234567894321_0001_000001"
);
SecretKey clientTokenMasterKey = clientToAMTokenMgr.CreateMasterKey(attemptId);
RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics
>();
Container container = new ContainerPBImpl();
container.SetId(ConverterUtils.ToContainerId("container_1234567891234_0001_01_000001"
));
RMAppAttempt mockAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(mockAttempt.GetAppAttemptId()).ThenReturn(attemptId);
Org.Mockito.Mockito.When(mockAttempt.GetMasterContainer()).ThenReturn(container);
Org.Mockito.Mockito.When(mockAttempt.GetClientTokenMasterKey()).ThenReturn(clientTokenMasterKey
);
Org.Mockito.Mockito.When(mockAttempt.GetRMAppAttemptMetrics()).ThenReturn(mockRmAppAttemptMetrics
);
Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()).
ThenReturn(new AggregateAppResourceUsage(0, 0));
store.StoreNewApplicationAttempt(mockAttempt);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
long finishTime = submitTime + 1000;
// Update attempt
ApplicationAttemptStateData newAttemptState = ApplicationAttemptStateData.NewInstance
(attemptId, container, store.GetCredentialsFromAppAttempt(mockAttempt), startTime
, RMAppAttemptState.Finished, "testUrl", "test", FinalApplicationStatus.Succeeded
, 100, finishTime, 0, 0);
store.UpdateApplicationAttemptState(newAttemptState);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// Update app
ApplicationStateData appState = ApplicationStateData.NewInstance(submitTime, startTime
, context, "test");
store.UpdateApplicationState(appState);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// Remove app
store.RemoveApplication(mockApp);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// store RM delegation token;
RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier(new Text("owner1"
), new Text("renewer1"), new Text("realuser1"));
long renewDate1 = Runtime.CurrentTimeMillis();
dtId1.SetSequenceNumber(1111);
store.StoreRMDelegationToken(dtId1, renewDate1);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
store.UpdateRMDelegationToken(dtId1, renewDate1);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// remove delegation key;
store.RemoveRMDelegationToken(dtId1);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// store delegation master key;
DelegationKey key = new DelegationKey(1234, 4321, Sharpen.Runtime.GetBytesForString
("keyBytes"));
store.StoreRMDTMasterKey(key);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// remove delegation master key;
store.RemoveRMDTMasterKey(key);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// store or update AMRMToken;
store.StoreOrUpdateAMRMTokenSecretManager(null, false);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
store.Close();
}