public virtual void TestNodeSingleContainerXML()
{
WebResource r = Resource();
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = new MockApp(1);
nmContext.GetApplications()[app.GetAppId()] = app;
Dictionary <string, string> hash = AddAppContainers(app);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app2 = new MockApp(2);
nmContext.GetApplications()[app2.GetAppId()] = app2;
AddAppContainers(app2);
foreach (string id in hash.Keys)
{
ClientResponse response = r.Path("ws").Path("v1").Path("node").Path("containers")
.Path(id).Accept(MediaType.ApplicationXml).Get <ClientResponse>();
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationXmlType, response.GetType());
string xml = response.GetEntity <string>();
DocumentBuilderFactory dbf = DocumentBuilderFactory.NewInstance();
DocumentBuilder db = dbf.NewDocumentBuilder();
InputSource @is = new InputSource();
@is.SetCharacterStream(new StringReader(xml));
Document dom = db.Parse(@is);
NodeList nodes = dom.GetElementsByTagName("container");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, nodes.GetLength
());
VerifyContainersInfoXML(nodes, nmContext.GetContainers()[ConverterUtils.ToContainerId
(id)]);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestMRAppMasterMaxAppAttempts()
{
// No matter what's the maxAppAttempt or attempt id, the isLastRetry always
// equals to false
bool[] expectedBools = new bool[] { false, false, false };
string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
string containerIdStr = "container_1317529182569_0004_000002_1";
string userName = "******";
ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId
(applicationAttemptIdStr);
ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr);
JobConf conf = new JobConf();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
FilePath stagingDir = new FilePath(MRApps.GetStagingAreaDir(conf, userName).ToString
());
stagingDir.Mkdirs();
for (int i = 0; i < expectedBools.Length; ++i)
{
MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId
, "host", -1, -1, Runtime.CurrentTimeMillis(), false, true);
MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName);
NUnit.Framework.Assert.AreEqual("isLastAMRetry is correctly computed.", expectedBools
[i], appMaster.IsLastAMRetry());
}
}
/// <exception cref="System.IO.IOException"/>
public static FileInputStream OpenLogFileForRead(string containerIdStr, FilePath
logFile, Context context)
{
ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr);
ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId
();
string user = context.GetApplications()[applicationId].GetUser();
try
{
return(SecureIOUtils.OpenForRead(logFile, user, null));
}
catch (IOException e)
{
if (e.Message.Contains("did not match expected owner '" + user + "'"))
{
Log.Error("Exception reading log file " + logFile.GetAbsolutePath(), e);
throw new IOException("Exception reading log file. Application submitted by '" +
user + "' doesn't own requested log file : " + logFile.GetName(), e);
}
else
{
throw new IOException("Exception reading log file. It might be because log " + "file was aggregated : "
+ logFile.GetName(), e);
}
}
}
/// <exception cref="System.Exception"/>
protected internal virtual ContainerId StoreAttempt(RMStateStore store, ApplicationAttemptId
attemptId, string containerIdStr, Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier
> appToken, SecretKey clientTokenMasterKey, RMStateStoreTestBase.TestDispatcher
dispatcher)
{
RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics
>();
Container container = new ContainerPBImpl();
container.SetId(ConverterUtils.ToContainerId(containerIdStr));
RMAppAttempt mockAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(mockAttempt.GetAppAttemptId()).ThenReturn(attemptId);
Org.Mockito.Mockito.When(mockAttempt.GetMasterContainer()).ThenReturn(container);
Org.Mockito.Mockito.When(mockAttempt.GetAMRMToken()).ThenReturn(appToken);
Org.Mockito.Mockito.When(mockAttempt.GetClientTokenMasterKey()).ThenReturn(clientTokenMasterKey
);
Org.Mockito.Mockito.When(mockAttempt.GetRMAppAttemptMetrics()).ThenReturn(mockRmAppAttemptMetrics
);
Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()).
ThenReturn(new AggregateAppResourceUsage(0, 0));
dispatcher.attemptId = attemptId;
store.StoreNewApplicationAttempt(mockAttempt);
WaitNotify(dispatcher);
return(container.GetId());
}
protected override void Render(HtmlBlock.Block html)
{
ContainerId containerID;
try
{
containerID = ConverterUtils.ToContainerId($(ContainerId));
}
catch (ArgumentException)
{
html.P().("Invalid containerId " + $(ContainerId)).();
return;
}
Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> div = html.Div("#content"
);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= this.nmContext.GetContainers()[containerID];
if (container == null)
{
div.H1("Unknown Container. Container might have completed, " + "please go back to the previous page and retry."
).();
return;
}
ContainerInfo info = new ContainerInfo(this.nmContext, container);
Info("Container information").("ContainerID", info.GetId()).("ContainerState", info
.GetState()).("ExitStatus", info.GetExitStatus()).("Diagnostics", info.GetDiagnostics
()).("User", info.GetUser()).("TotalMemoryNeeded", info.GetMemoryNeeded()).("TotalVCoresNeeded"
, info.GetVCoresNeeded()).("logs", info.GetShortLogLink(), "Link to logs");
html.(typeof(InfoBlock));
}
/// <exception cref="System.IO.IOException"/>
public virtual IDictionary <ContainerId, ContainerHistoryData> GetContainers(ApplicationAttemptId
appAttemptId)
{
IDictionary <ContainerId, ContainerHistoryData> historyDataMap = new Dictionary <ContainerId
, ContainerHistoryData>();
FileSystemApplicationHistoryStore.HistoryFileReader hfReader = GetHistoryFileReader
(appAttemptId.GetApplicationId());
try
{
while (hfReader.HasNext())
{
FileSystemApplicationHistoryStore.HistoryFileReader.Entry entry = hfReader.Next();
if (entry.key.id.StartsWith(ConverterUtils.ContainerPrefix))
{
ContainerId containerId = ConverterUtils.ToContainerId(entry.key.id);
if (containerId.GetApplicationAttemptId().Equals(appAttemptId))
{
ContainerHistoryData historyData = historyDataMap[containerId];
if (historyData == null)
{
historyData = ContainerHistoryData.NewInstance(containerId, null, null, null, long.MinValue
, long.MaxValue, null, int.MaxValue, null);
historyDataMap[containerId] = historyData;
}
if (entry.key.suffix.Equals(StartDataSuffix))
{
MergeContainerHistoryData(historyData, ParseContainerStartData(entry.value));
}
else
{
if (entry.key.suffix.Equals(FinishDataSuffix))
{
MergeContainerHistoryData(historyData, ParseContainerFinishData(entry.value));
}
}
}
}
}
Log.Info("Completed reading history information of all conatiners" + " of application attempt "
+ appAttemptId);
}
catch (IOException)
{
Log.Info("Error when reading history information of some containers" + " of application attempt "
+ appAttemptId);
}
finally
{
hfReader.Close();
}
return(historyDataMap);
}
protected internal static ContainerId ParseContainerId(string containerId)
{
if (containerId == null || containerId.IsEmpty())
{
throw new NotFoundException("containerId, " + containerId + ", is empty or null");
}
ContainerId cid = ConverterUtils.ToContainerId(containerId);
if (cid == null)
{
throw new NotFoundException("containerId is null");
}
return(cid);
}
/// <exception cref="System.IO.IOException"/>
private static void LoadContainerToken(NMStateStoreService.RecoveredContainerTokensState
state, string key, string containerIdStr, byte[] value)
{
ContainerId containerId;
long expTime;
try
{
containerId = ConverterUtils.ToContainerId(containerIdStr);
expTime = long.Parse(JniDBFactory.AsString(value));
}
catch (ArgumentException e)
{
throw new IOException("Bad container token state for " + key, e);
}
state.activeTokens[containerId] = expTime;
}
public virtual void TestMRAppMasterForDifferentUser()
{
string applicationAttemptIdStr = "appattempt_1317529182569_0004_000001";
string containerIdStr = "container_1317529182569_0004_000001_1";
string userName = "******";
ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId
(applicationAttemptIdStr);
ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr);
MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId
, "host", -1, -1, Runtime.CurrentTimeMillis());
JobConf conf = new JobConf();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName);
Path userPath = new Path(stagingDir, userName);
Path userStagingPath = new Path(userPath, ".staging");
NUnit.Framework.Assert.AreEqual(userStagingPath.ToString(), appMaster.stagingDirPath
.ToString());
}
/// <exception cref="System.IO.IOException"/>
private static void ConfigureTask(JobConf job, Task task, Credentials credentials
, Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jt)
{
job.SetCredentials(credentials);
ApplicationAttemptId appAttemptId = ConverterUtils.ToContainerId(Runtime.Getenv(ApplicationConstants.Environment
.ContainerId.ToString())).GetApplicationAttemptId();
Log.Debug("APPLICATION_ATTEMPT_ID: " + appAttemptId);
// Set it in conf, so as to be able to be used the the OutputCommitter.
job.SetInt(MRJobConfig.ApplicationAttemptId, appAttemptId.GetAttemptId());
// set tcp nodelay
job.SetBoolean("ipc.client.tcpnodelay", true);
job.SetClass(MRConfig.TaskLocalOutputClass, typeof(YarnOutputFiles), typeof(MapOutputFile
));
// set the jobToken and shuffle secrets into task
task.SetJobTokenSecret(JobTokenSecretManager.CreateSecretKey(jt.GetPassword()));
byte[] shuffleSecret = TokenCache.GetShuffleSecretKey(credentials);
if (shuffleSecret == null)
{
Log.Warn("Shuffle secret missing from task credentials." + " Using job token secret as shuffle secret."
);
shuffleSecret = jt.GetPassword();
}
task.SetShuffleSecret(JobTokenSecretManager.CreateSecretKey(shuffleSecret));
// setup the child's MRConfig.LOCAL_DIR.
ConfigureLocalDirs(task, job);
// setup the child's attempt directories
// Do the task-type specific localization
task.LocalizeConfiguration(job);
// Set up the DistributedCache related configs
MRApps.SetupDistributedCacheLocal(job);
// Overwrite the localized task jobconf which is linked to in the current
// work-dir.
Path localTaskFile = new Path(MRJobConfig.JobConfFile);
WriteLocalJobFile(localTaskFile, job);
task.SetJobFile(localTaskFile.ToString());
task.SetConf(job);
}
public virtual Response GetLogs(string containerIdStr, string filename)
{
ContainerId containerId;
try
{
containerId = ConverterUtils.ToContainerId(containerIdStr);
}
catch (ArgumentException)
{
return(Response.Status(Response.Status.BadRequest).Build());
}
FilePath logFile = null;
try
{
logFile = ContainerLogsUtils.GetContainerLogFile(containerId, filename, request.GetRemoteUser
(), nmContext);
}
catch (NotFoundException ex)
{
return(Response.Status(Response.Status.NotFound).Entity(ex.Message).Build());
}
catch (YarnException ex)
{
return(Response.ServerError().Entity(ex.Message).Build());
}
try
{
FileInputStream fis = ContainerLogsUtils.OpenLogFileForRead(containerIdStr, logFile
, nmContext);
StreamingOutput stream = new _StreamingOutput_240(fis);
return(Response.Ok(stream).Build());
}
catch (IOException ex)
{
return(Response.ServerError().Entity(ex.Message).Build());
}
}
public virtual ContainerInfo GetNodeContainer(string id)
{
ContainerId containerId = null;
Init();
try
{
containerId = ConverterUtils.ToContainerId(id);
}
catch (Exception)
{
throw new BadRequestException("invalid container id, " + id);
}
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= nmContext.GetContainers()[containerId];
if (container == null)
{
throw new NotFoundException("container with id, " + id + ", not found");
}
return(new ContainerInfo(this.nmContext, container, uriInfo.GetBaseUri().ToString
(), webapp.Name()));
}
public virtual void TestMRAppMasterSuccessLock()
{
string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
string containerIdStr = "container_1317529182569_0004_000002_1";
string userName = "******";
JobConf conf = new JobConf();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId
(applicationAttemptIdStr);
JobId jobId = TypeConverter.ToYarn(TypeConverter.FromYarn(applicationAttemptId.GetApplicationId
()));
Path start = MRApps.GetStartJobCommitFile(conf, userName, jobId);
Path end = MRApps.GetEndJobCommitSuccessFile(conf, userName, jobId);
FileSystem fs = FileSystem.Get(conf);
fs.Create(start).Close();
fs.Create(end).Close();
ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr);
MRAppMaster appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host"
, -1, -1, Runtime.CurrentTimeMillis(), false, false);
bool caught = false;
try
{
MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName);
}
catch (IOException e)
{
//The IO Exception is expected
Log.Info("Caught expected Exception", e);
caught = true;
}
NUnit.Framework.Assert.IsTrue(caught);
NUnit.Framework.Assert.IsTrue(appMaster.errorHappenedShutDown);
NUnit.Framework.Assert.AreEqual(JobStateInternal.Succeeded, appMaster.forcedState
);
appMaster.Stop();
// verify the final status is SUCCEEDED
VerifyFailedStatus((MRAppMasterTest)appMaster, "SUCCEEDED");
}
private ContainerId VerifyAndGetContainerId(HtmlBlock.Block html)
{
string containerIdStr = $(YarnWebParams.ContainerId);
if (containerIdStr == null || containerIdStr.IsEmpty())
{
html.H1().("Cannot get container logs without a ContainerId").();
return(null);
}
ContainerId containerId = null;
try
{
containerId = ConverterUtils.ToContainerId(containerIdStr);
}
catch (ArgumentException)
{
html.H1().("Cannot get container logs for invalid containerId: " + containerIdStr
).();
return(null);
}
return(containerId);
}
public virtual void Logs()
{
string containerIdStr = $(ContainerId);
ContainerId containerId = null;
try
{
containerId = ConverterUtils.ToContainerId(containerIdStr);
}
catch (ArgumentException)
{
Render(typeof(ContainerLogsPage));
return;
}
ApplicationId appId = containerId.GetApplicationAttemptId().GetApplicationId();
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = nmContext.GetApplications()[appId];
if (app == null && nmConf.GetBoolean(YarnConfiguration.LogAggregationEnabled, YarnConfiguration
.DefaultLogAggregationEnabled))
{
string logServerUrl = nmConf.Get(YarnConfiguration.YarnLogServerUrl);
string redirectUrl = null;
if (logServerUrl == null || logServerUrl.IsEmpty())
{
redirectUrl = "false";
}
else
{
redirectUrl = Url(logServerUrl, nmContext.GetNodeId().ToString(), containerIdStr,
containerIdStr, $(AppOwner));
}
Set(ContainerLogsPage.RedirectUrl, redirectUrl);
}
Render(typeof(ContainerLogsPage));
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
/// <exception cref="System.Exception"/>
public virtual void TestNodeSingleContainersHelper(string media)
{
WebResource r = Resource();
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = new MockApp(1);
nmContext.GetApplications()[app.GetAppId()] = app;
Dictionary <string, string> hash = AddAppContainers(app);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app2 = new MockApp(2);
nmContext.GetApplications()[app2.GetAppId()] = app2;
AddAppContainers(app2);
foreach (string id in hash.Keys)
{
ClientResponse response = r.Path("ws").Path("v1").Path("node").Path("containers")
.Path(id).Accept(media).Get <ClientResponse>();
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
);
JSONObject json = response.GetEntity <JSONObject>();
VerifyNodeContainerInfo(json.GetJSONObject("container"), nmContext.GetContainers(
)[ConverterUtils.ToContainerId(id)]);
}
}
public virtual void TestMRAppMasterMissingStaging()
{
string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
string containerIdStr = "container_1317529182569_0004_000002_1";
string userName = "******";
JobConf conf = new JobConf();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId
(applicationAttemptIdStr);
//Delete the staging directory
FilePath dir = new FilePath(stagingDir);
if (dir.Exists())
{
FileUtils.DeleteDirectory(dir);
}
ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr);
MRAppMaster appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host"
, -1, -1, Runtime.CurrentTimeMillis(), false, false);
bool caught = false;
try
{
MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName);
}
catch (IOException e)
{
//The IO Exception is expected
Log.Info("Caught expected Exception", e);
caught = true;
}
NUnit.Framework.Assert.IsTrue(caught);
NUnit.Framework.Assert.IsTrue(appMaster.errorHappenedShutDown);
//Copying the history file is disabled, but it is not really visible from
//here
NUnit.Framework.Assert.AreEqual(JobStateInternal.Error, appMaster.forcedState);
appMaster.Stop();
}
protected override void Render(HtmlBlock.Block html)
{
string redirectUrl = $(RedirectUrl);
if (redirectUrl != null && redirectUrl.Equals("false"))
{
html.H1("Failed while trying to construct the redirect url to the log" + " server. Log Server url may not be configured"
);
}
//Intentional fallthrough.
ContainerId containerId;
try
{
containerId = ConverterUtils.ToContainerId($(YarnWebParams.ContainerId));
}
catch (ArgumentException)
{
html.H1("Invalid container ID: " + $(YarnWebParams.ContainerId));
return;
}
try
{
if ($(ContainerLogType).IsEmpty())
{
IList <FilePath> logFiles = ContainerLogsUtils.GetContainerLogDirs(containerId, Request
().GetRemoteUser(), nmContext);
PrintLogFileDirectory(html, logFiles);
}
else
{
FilePath logFile = ContainerLogsUtils.GetContainerLogFile(containerId, $(ContainerLogType
), Request().GetRemoteUser(), nmContext);
PrintLogFile(html, logFile);
}
}
/// <summary>Get the process id from specified file path.</summary>
/// <remarks>
/// Get the process id from specified file path.
/// Parses each line to find a valid number
/// and returns the first one found.
/// </remarks>
/// <returns>Process Id if obtained from path specified else null</returns>
/// <exception cref="System.IO.IOException"/>
public static string GetProcessId(Path path)
{
if (path == null)
{
throw new IOException("Trying to access process id from a null path");
}
Log.Debug("Accessing pid from pid file " + path);
string processId = null;
BufferedReader bufReader = null;
try
{
FilePath file = new FilePath(path.ToString());
if (file.Exists())
{
FileInputStream fis = new FileInputStream(file);
bufReader = new BufferedReader(new InputStreamReader(fis, "UTF-8"));
while (true)
{
string line = bufReader.ReadLine();
if (line == null)
{
break;
}
string temp = line.Trim();
if (!temp.IsEmpty())
{
if (Shell.Windows)
{
// On Windows, pid is expected to be a container ID, so find first
// line that parses successfully as a container ID.
try
{
ConverterUtils.ToContainerId(temp);
processId = temp;
break;
}
catch (Exception)
{
}
}
else
{
// do nothing
// Otherwise, find first line containing a numeric pid.
try
{
long pid = Sharpen.Extensions.ValueOf(temp);
if (pid > 0)
{
processId = temp;
break;
}
}
catch (Exception)
{
}
}
}
}
}
}
finally
{
// do nothing
if (bufReader != null)
{
bufReader.Close();
}
}
Log.Debug("Got pid " + (processId != null ? processId : "null") + " from path " +
path);
return(processId);
}
public virtual void TestFencedState()
{
TestZKRMStateStore.TestZKRMStateStoreTester zkTester = new TestZKRMStateStore.TestZKRMStateStoreTester
(this);
RMStateStore store = zkTester.GetRMStateStore();
// Move state to FENCED from ACTIVE
store.UpdateFencedState();
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
long submitTime = Runtime.CurrentTimeMillis();
long startTime = submitTime + 1000;
// Add a new app
RMApp mockApp = Org.Mockito.Mockito.Mock <RMApp>();
ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();
Org.Mockito.Mockito.When(mockApp.GetSubmitTime()).ThenReturn(submitTime);
Org.Mockito.Mockito.When(mockApp.GetStartTime()).ThenReturn(startTime);
Org.Mockito.Mockito.When(mockApp.GetApplicationSubmissionContext()).ThenReturn(context
);
Org.Mockito.Mockito.When(mockApp.GetUser()).ThenReturn("test");
store.StoreNewApplication(mockApp);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// Add a new attempt
ClientToAMTokenSecretManagerInRM clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM
();
ApplicationAttemptId attemptId = ConverterUtils.ToApplicationAttemptId("appattempt_1234567894321_0001_000001"
);
SecretKey clientTokenMasterKey = clientToAMTokenMgr.CreateMasterKey(attemptId);
RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics
>();
Container container = new ContainerPBImpl();
container.SetId(ConverterUtils.ToContainerId("container_1234567891234_0001_01_000001"
));
RMAppAttempt mockAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(mockAttempt.GetAppAttemptId()).ThenReturn(attemptId);
Org.Mockito.Mockito.When(mockAttempt.GetMasterContainer()).ThenReturn(container);
Org.Mockito.Mockito.When(mockAttempt.GetClientTokenMasterKey()).ThenReturn(clientTokenMasterKey
);
Org.Mockito.Mockito.When(mockAttempt.GetRMAppAttemptMetrics()).ThenReturn(mockRmAppAttemptMetrics
);
Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()).
ThenReturn(new AggregateAppResourceUsage(0, 0));
store.StoreNewApplicationAttempt(mockAttempt);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
long finishTime = submitTime + 1000;
// Update attempt
ApplicationAttemptStateData newAttemptState = ApplicationAttemptStateData.NewInstance
(attemptId, container, store.GetCredentialsFromAppAttempt(mockAttempt), startTime
, RMAppAttemptState.Finished, "testUrl", "test", FinalApplicationStatus.Succeeded
, 100, finishTime, 0, 0);
store.UpdateApplicationAttemptState(newAttemptState);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// Update app
ApplicationStateData appState = ApplicationStateData.NewInstance(submitTime, startTime
, context, "test");
store.UpdateApplicationState(appState);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// Remove app
store.RemoveApplication(mockApp);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// store RM delegation token;
RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier(new Text("owner1"
), new Text("renewer1"), new Text("realuser1"));
long renewDate1 = Runtime.CurrentTimeMillis();
dtId1.SetSequenceNumber(1111);
store.StoreRMDelegationToken(dtId1, renewDate1);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
store.UpdateRMDelegationToken(dtId1, renewDate1);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// remove delegation key;
store.RemoveRMDelegationToken(dtId1);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// store delegation master key;
DelegationKey key = new DelegationKey(1234, 4321, Sharpen.Runtime.GetBytesForString
("keyBytes"));
store.StoreRMDTMasterKey(key);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// remove delegation master key;
store.RemoveRMDTMasterKey(key);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
// store or update AMRMToken;
store.StoreOrUpdateAMRMTokenSecretManager(null, false);
NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
true, store.IsFencedState());
store.Close();
}
// TODO later: add explicit "isUber()" checks of some sort
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
public virtual void TestContainerRollingLog()
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
SleepJob sleepJob = new SleepJob();
JobConf sleepConf = new JobConf(mrCluster.GetConfig());
sleepConf.Set(MRJobConfig.MapLogLevel, Level.All.ToString());
long userLogKb = 4;
sleepConf.SetLong(MRJobConfig.TaskUserlogLimit, userLogKb);
sleepConf.SetInt(MRJobConfig.TaskLogBackups, 3);
sleepConf.Set(MRJobConfig.MrAmLogLevel, Level.All.ToString());
long amLogKb = 7;
sleepConf.SetLong(MRJobConfig.MrAmLogKb, amLogKb);
sleepConf.SetInt(MRJobConfig.MrAmLogBackups, 7);
sleepJob.SetConf(sleepConf);
Job job = sleepJob.CreateJob(1, 0, 1L, 100, 0L, 0);
job.SetJarByClass(typeof(SleepJob));
job.AddFileToClassPath(AppJar);
// The AppMaster jar itself.
job.WaitForCompletion(true);
JobId jobId = TypeConverter.ToYarn(job.GetJobID());
ApplicationId appID = jobId.GetAppId();
int pollElapsed = 0;
while (true)
{
Sharpen.Thread.Sleep(1000);
pollElapsed += 1000;
if (TerminalRmAppStates.Contains(mrCluster.GetResourceManager().GetRMContext().GetRMApps
()[appID].GetState()))
{
break;
}
if (pollElapsed >= 60000)
{
Log.Warn("application did not reach terminal state within 60 seconds");
break;
}
}
NUnit.Framework.Assert.AreEqual(RMAppState.Finished, mrCluster.GetResourceManager
().GetRMContext().GetRMApps()[appID].GetState());
// Job finished, verify logs
//
string appIdStr = appID.ToString();
string appIdSuffix = Sharpen.Runtime.Substring(appIdStr, "application_".Length, appIdStr
.Length);
string containerGlob = "container_" + appIdSuffix + "_*_*";
string syslogGlob = appIdStr + Path.Separator + containerGlob + Path.Separator +
TaskLog.LogName.Syslog;
int numAppMasters = 0;
int numMapTasks = 0;
for (int i = 0; i < NumNodeMgrs; i++)
{
Configuration nmConf = mrCluster.GetNodeManager(i).GetConfig();
foreach (string logDir in nmConf.GetTrimmedStrings(YarnConfiguration.NmLogDirs))
{
Path absSyslogGlob = new Path(logDir + Path.Separator + syslogGlob);
Log.Info("Checking for glob: " + absSyslogGlob);
FileStatus[] syslogs = localFs.GlobStatus(absSyslogGlob);
foreach (FileStatus slog in syslogs)
{
bool foundAppMaster = job.IsUber();
Path containerPathComponent = slog.GetPath().GetParent();
if (!foundAppMaster)
{
ContainerId cid = ConverterUtils.ToContainerId(containerPathComponent.GetName());
foundAppMaster = ((cid.GetContainerId() & ContainerId.ContainerIdBitmask) == 1);
}
FileStatus[] sysSiblings = localFs.GlobStatus(new Path(containerPathComponent, TaskLog.LogName
.Syslog + "*"));
// sort to ensure for i > 0 sysSiblings[i] == "syslog.i"
Arrays.Sort(sysSiblings);
if (foundAppMaster)
{
numAppMasters++;
}
else
{
numMapTasks++;
}
if (foundAppMaster)
{
NUnit.Framework.Assert.AreSame("Unexpected number of AM sylog* files", sleepConf.
GetInt(MRJobConfig.MrAmLogBackups, 0) + 1, sysSiblings.Length);
NUnit.Framework.Assert.IsTrue("AM syslog.1 length kb should be >= " + amLogKb, sysSiblings
[1].GetLen() >= amLogKb * 1024);
}
else
{
NUnit.Framework.Assert.AreSame("Unexpected number of MR task sylog* files", sleepConf
.GetInt(MRJobConfig.TaskLogBackups, 0) + 1, sysSiblings.Length);
NUnit.Framework.Assert.IsTrue("MR syslog.1 length kb should be >= " + userLogKb,
sysSiblings[1].GetLen() >= userLogKb * 1024);
}
}
}
}
// Make sure we checked non-empty set
//
NUnit.Framework.Assert.AreEqual("No AppMaster log found!", 1, numAppMasters);
if (sleepConf.GetBoolean(MRJobConfig.JobUbertaskEnable, false))
{
NUnit.Framework.Assert.AreEqual("MapTask log with uber found!", 0, numMapTasks);
}
else
{
NUnit.Framework.Assert.AreEqual("No MapTask log found!", 1, numMapTasks);
}
}
private static ApplicationAttemptReport ConvertToApplicationAttemptReport(TimelineEntity
entity)
{
string host = null;
int rpcPort = -1;
ContainerId amContainerId = null;
string trackingUrl = null;
string originalTrackingUrl = null;
string diagnosticsInfo = null;
YarnApplicationAttemptState state = null;
IList <TimelineEvent> events = entity.GetEvents();
if (events != null)
{
foreach (TimelineEvent @event in events)
{
if (@event.GetEventType().Equals(AppAttemptMetricsConstants.RegisteredEventType))
{
IDictionary <string, object> eventInfo = @event.GetEventInfo();
if (eventInfo == null)
{
continue;
}
if (eventInfo.Contains(AppAttemptMetricsConstants.HostEventInfo))
{
host = eventInfo[AppAttemptMetricsConstants.HostEventInfo].ToString();
}
if (eventInfo.Contains(AppAttemptMetricsConstants.RpcPortEventInfo))
{
rpcPort = (int)eventInfo[AppAttemptMetricsConstants.RpcPortEventInfo];
}
if (eventInfo.Contains(AppAttemptMetricsConstants.MasterContainerEventInfo))
{
amContainerId = ConverterUtils.ToContainerId(eventInfo[AppAttemptMetricsConstants
.MasterContainerEventInfo].ToString());
}
}
else
{
if (@event.GetEventType().Equals(AppAttemptMetricsConstants.FinishedEventType))
{
IDictionary <string, object> eventInfo = @event.GetEventInfo();
if (eventInfo == null)
{
continue;
}
if (eventInfo.Contains(AppAttemptMetricsConstants.TrackingUrlEventInfo))
{
trackingUrl = eventInfo[AppAttemptMetricsConstants.TrackingUrlEventInfo].ToString
();
}
if (eventInfo.Contains(AppAttemptMetricsConstants.OriginalTrackingUrlEventInfo))
{
originalTrackingUrl = eventInfo[AppAttemptMetricsConstants.OriginalTrackingUrlEventInfo
].ToString();
}
if (eventInfo.Contains(AppAttemptMetricsConstants.DiagnosticsInfoEventInfo))
{
diagnosticsInfo = eventInfo[AppAttemptMetricsConstants.DiagnosticsInfoEventInfo].
ToString();
}
if (eventInfo.Contains(AppAttemptMetricsConstants.StateEventInfo))
{
state = YarnApplicationAttemptState.ValueOf(eventInfo[AppAttemptMetricsConstants.
StateEventInfo].ToString());
}
}
}
}
}
return(ApplicationAttemptReport.NewInstance(ConverterUtils.ToApplicationAttemptId
(entity.GetEntityId()), host, rpcPort, trackingUrl, originalTrackingUrl, diagnosticsInfo
, state, amContainerId));
}
/// <returns>the ContainerId for the MRAppMaster.</returns>
public virtual ContainerId GetContainerId()
{
return(ConverterUtils.ToContainerId(datum.containerId.ToString()));
}
private static ContainerReport ConvertToContainerReport(TimelineEntity entity, string
serverHttpAddress, string user)
{
int allocatedMem = 0;
int allocatedVcore = 0;
string allocatedHost = null;
int allocatedPort = -1;
int allocatedPriority = 0;
long createdTime = 0;
long finishedTime = 0;
string diagnosticsInfo = null;
int exitStatus = ContainerExitStatus.Invalid;
ContainerState state = null;
string nodeHttpAddress = null;
IDictionary <string, object> entityInfo = entity.GetOtherInfo();
if (entityInfo != null)
{
if (entityInfo.Contains(ContainerMetricsConstants.AllocatedMemoryEntityInfo))
{
allocatedMem = (int)entityInfo[ContainerMetricsConstants.AllocatedMemoryEntityInfo
];
}
if (entityInfo.Contains(ContainerMetricsConstants.AllocatedVcoreEntityInfo))
{
allocatedVcore = (int)entityInfo[ContainerMetricsConstants.AllocatedVcoreEntityInfo
];
}
if (entityInfo.Contains(ContainerMetricsConstants.AllocatedHostEntityInfo))
{
allocatedHost = entityInfo[ContainerMetricsConstants.AllocatedHostEntityInfo].ToString
();
}
if (entityInfo.Contains(ContainerMetricsConstants.AllocatedPortEntityInfo))
{
allocatedPort = (int)entityInfo[ContainerMetricsConstants.AllocatedPortEntityInfo
];
}
if (entityInfo.Contains(ContainerMetricsConstants.AllocatedPriorityEntityInfo))
{
allocatedPriority = (int)entityInfo[ContainerMetricsConstants.AllocatedPriorityEntityInfo
];
}
if (entityInfo.Contains(ContainerMetricsConstants.AllocatedHostHttpAddressEntityInfo
))
{
nodeHttpAddress = (string)entityInfo[ContainerMetricsConstants.AllocatedHostHttpAddressEntityInfo
];
}
}
IList <TimelineEvent> events = entity.GetEvents();
if (events != null)
{
foreach (TimelineEvent @event in events)
{
if (@event.GetEventType().Equals(ContainerMetricsConstants.CreatedEventType))
{
createdTime = @event.GetTimestamp();
}
else
{
if (@event.GetEventType().Equals(ContainerMetricsConstants.FinishedEventType))
{
finishedTime = @event.GetTimestamp();
IDictionary <string, object> eventInfo = @event.GetEventInfo();
if (eventInfo == null)
{
continue;
}
if (eventInfo.Contains(ContainerMetricsConstants.DiagnosticsInfoEventInfo))
{
diagnosticsInfo = eventInfo[ContainerMetricsConstants.DiagnosticsInfoEventInfo].ToString
();
}
if (eventInfo.Contains(ContainerMetricsConstants.ExitStatusEventInfo))
{
exitStatus = (int)eventInfo[ContainerMetricsConstants.ExitStatusEventInfo];
}
if (eventInfo.Contains(ContainerMetricsConstants.StateEventInfo))
{
state = ContainerState.ValueOf(eventInfo[ContainerMetricsConstants.StateEventInfo
].ToString());
}
}
}
}
}
NodeId allocatedNode = NodeId.NewInstance(allocatedHost, allocatedPort);
ContainerId containerId = ConverterUtils.ToContainerId(entity.GetEntityId());
string logUrl = WebAppUtils.GetAggregatedLogURL(serverHttpAddress, allocatedNode.
ToString(), containerId.ToString(), containerId.ToString(), user);
return(ContainerReport.NewInstance(ConverterUtils.ToContainerId(entity.GetEntityId
()), Resource.NewInstance(allocatedMem, allocatedVcore), NodeId.NewInstance(allocatedHost
, allocatedPort), Priority.NewInstance(allocatedPriority), createdTime, finishedTime
, diagnosticsInfo, logUrl, exitStatus, state, nodeHttpAddress));
}
/// <exception cref="System.IO.IOException"/>
public override IList <NMStateStoreService.RecoveredContainerState> LoadContainersState
()
{
AList <NMStateStoreService.RecoveredContainerState> containers = new AList <NMStateStoreService.RecoveredContainerState
>();
AList <ContainerId> containersToRemove = new AList <ContainerId>();
LeveldbIterator iter = null;
try
{
iter = new LeveldbIterator(db);
iter.Seek(JniDBFactory.Bytes(ContainersKeyPrefix));
while (iter.HasNext())
{
KeyValuePair <byte[], byte[]> entry = iter.PeekNext();
string key = JniDBFactory.AsString(entry.Key);
if (!key.StartsWith(ContainersKeyPrefix))
{
break;
}
int idEndPos = key.IndexOf('/', ContainersKeyPrefix.Length);
if (idEndPos < 0)
{
throw new IOException("Unable to determine container in key: " + key);
}
ContainerId containerId = ConverterUtils.ToContainerId(Sharpen.Runtime.Substring(
key, ContainersKeyPrefix.Length, idEndPos));
string keyPrefix = Sharpen.Runtime.Substring(key, 0, idEndPos + 1);
NMStateStoreService.RecoveredContainerState rcs = LoadContainerState(containerId,
iter, keyPrefix);
// Don't load container without StartContainerRequest
if (rcs.startRequest != null)
{
containers.AddItem(rcs);
}
else
{
containersToRemove.AddItem(containerId);
}
}
}
catch (DBException e)
{
throw new IOException(e);
}
finally
{
if (iter != null)
{
iter.Close();
}
}
// remove container without StartContainerRequest
foreach (ContainerId containerId_1 in containersToRemove)
{
Log.Warn("Remove container " + containerId_1 + " with incomplete records");
try
{
RemoveContainer(containerId_1);
}
catch (IOException e)
{
// TODO: kill and cleanup the leaked container
Log.Error("Unable to remove container " + containerId_1 + " in store", e);
}
}
return(containers);
}
protected override void Render(HtmlBlock.Block html)
{
string containerid = $(YarnWebParams.ContainerId);
if (containerid.IsEmpty())
{
Puts("Bad request: requires container ID");
return;
}
ContainerId containerId = null;
try
{
containerId = ConverterUtils.ToContainerId(containerid);
}
catch (ArgumentException)
{
Puts("Invalid container ID: " + containerid);
return;
}
UserGroupInformation callerUGI = GetCallerUGI();
ContainerReport containerReport = null;
try
{
GetContainerReportRequest request = GetContainerReportRequest.NewInstance(containerId
);
if (callerUGI == null)
{
containerReport = appBaseProt.GetContainerReport(request).GetContainerReport();
}
else
{
containerReport = callerUGI.DoAs(new _PrivilegedExceptionAction_78(this, request)
);
}
}
catch (Exception e)
{
string message = "Failed to read the container " + containerid + ".";
Log.Error(message, e);
html.P().(message).();
return;
}
if (containerReport == null)
{
Puts("Container not found: " + containerid);
return;
}
ContainerInfo container = new ContainerInfo(containerReport);
SetTitle(StringHelper.Join("Container ", containerid));
Info("Container Overview").("Container State:", container.GetContainerState() ==
null ? Unavailable : container.GetContainerState()).("Exit Status:", container.GetContainerExitStatus
()).("Node:", container.GetNodeHttpAddress() == null ? "#" : container.GetNodeHttpAddress
(), container.GetNodeHttpAddress() == null ? "N/A" : container.GetNodeHttpAddress
()).("Priority:", container.GetPriority()).("Started:", Times.Format(container.GetStartedTime
())).("Elapsed:", StringUtils.FormatTime(Times.Elapsed(container.GetStartedTime(
), container.GetFinishedTime()))).("Resource:", container.GetAllocatedMB() + " Memory, "
+ container.GetAllocatedVCores() + " VCores").("Logs:", container.GetLogUrl() ==
null ? "#" : container.GetLogUrl(), container.GetLogUrl() == null ? "N/A" : "Logs"
).("Diagnostics:", container.GetDiagnosticsInfo() == null ? string.Empty : container
.GetDiagnosticsInfo());
html.(typeof(InfoBlock));
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
/// <exception cref="System.Exception"/>
public virtual void TestNodeHelper(string path, string media)
{
WebResource r = Resource();
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = new MockApp(1);
nmContext.GetApplications()[app.GetAppId()] = app;
AddAppContainers(app);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app2 = new MockApp(2);
nmContext.GetApplications()[app2.GetAppId()] = app2;
AddAppContainers(app2);
ClientResponse response = r.Path("ws").Path("v1").Path("node").Path(path).Accept(
media).Get <ClientResponse>();
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
);
JSONObject json = response.GetEntity <JSONObject>();
JSONObject info = json.GetJSONObject("containers");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, info.Length());
JSONArray conInfo = info.GetJSONArray("container");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 4, conInfo.Length
());
for (int i = 0; i < conInfo.Length(); i++)
{
VerifyNodeContainerInfo(conInfo.GetJSONObject(i), nmContext.GetContainers()[ConverterUtils
.ToContainerId(conInfo.GetJSONObject(i).GetString("id"))]);
}
}
/// <summary>Prints the container report for an container id.</summary>
/// <param name="containerId"/>
/// <returns>exitCode</returns>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private int PrintContainerReport(string containerId)
{
ContainerReport containerReport = null;
try
{
containerReport = client.GetContainerReport((ConverterUtils.ToContainerId(containerId
)));
}
catch (ApplicationNotFoundException)
{
sysout.WriteLine("Application for Container with id '" + containerId + "' doesn't exist in RM or Timeline Server."
);
return(-1);
}
catch (ApplicationAttemptNotFoundException)
{
sysout.WriteLine("Application Attempt for Container with id '" + containerId + "' doesn't exist in RM or Timeline Server."
);
return(-1);
}
catch (ContainerNotFoundException)
{
sysout.WriteLine("Container with id '" + containerId + "' doesn't exist in RM or Timeline Server."
);
return(-1);
}
// Use PrintWriter.println, which uses correct platform line ending.
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter containerReportStr = new PrintWriter(new OutputStreamWriter(baos, Sharpen.Extensions.GetEncoding
("UTF-8")));
if (containerReport != null)
{
containerReportStr.WriteLine("Container Report : ");
containerReportStr.Write("\tContainer-Id : ");
containerReportStr.WriteLine(containerReport.GetContainerId());
containerReportStr.Write("\tStart-Time : ");
containerReportStr.WriteLine(containerReport.GetCreationTime());
containerReportStr.Write("\tFinish-Time : ");
containerReportStr.WriteLine(containerReport.GetFinishTime());
containerReportStr.Write("\tState : ");
containerReportStr.WriteLine(containerReport.GetContainerState());
containerReportStr.Write("\tLOG-URL : ");
containerReportStr.WriteLine(containerReport.GetLogUrl());
containerReportStr.Write("\tHost : ");
containerReportStr.WriteLine(containerReport.GetAssignedNode());
containerReportStr.Write("\tNodeHttpAddress : ");
containerReportStr.WriteLine(containerReport.GetNodeHttpAddress() == null ? "N/A"
: containerReport.GetNodeHttpAddress());
containerReportStr.Write("\tDiagnostics : ");
containerReportStr.Write(containerReport.GetDiagnosticsInfo());
}
else
{
containerReportStr.Write("Container with id '" + containerId + "' doesn't exist in Timeline Server."
);
containerReportStr.Close();
sysout.WriteLine(baos.ToString("UTF-8"));
return(-1);
}
containerReportStr.Close();
sysout.WriteLine(baos.ToString("UTF-8"));
return(0);
}
public TaskAttemptStartedEvent(TaskAttemptID attemptId, TaskType taskType, long startTime
, string trackerName, int httpPort, int shufflePort, string locality, string avataar
)
: this(attemptId, taskType, startTime, trackerName, httpPort, shufflePort, ConverterUtils
.ToContainerId("container_-1_-1_-1_-1"), locality, avataar)
{
}