/// <summary>This tests whether a containerId is serialized/deserialized with epoch.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private void TestContainerTokenWithEpoch(Configuration conf)
{
Log.Info("Running test for serializing/deserializing containerIds");
NMTokenSecretManagerInRM nmTokenSecretManagerInRM = yarnCluster.GetResourceManager
().GetRMContext().GetNMTokenSecretManager();
ApplicationId appId = ApplicationId.NewInstance(1, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 0);
ContainerId cId = ContainerId.NewContainerId(appAttemptId, (5L << 40) | 3L);
NodeManager nm = yarnCluster.GetNodeManager(0);
NMTokenSecretManagerInNM nmTokenSecretManagerInNM = nm.GetNMContext().GetNMTokenSecretManager
();
string user = "******";
WaitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm);
NodeId nodeId = nm.GetNMContext().GetNodeId();
// Both id should be equal.
NUnit.Framework.Assert.AreEqual(nmTokenSecretManagerInNM.GetCurrentKey().GetKeyId
(), nmTokenSecretManagerInRM.GetCurrentKey().GetKeyId());
// Creating a normal Container Token
RMContainerTokenSecretManager containerTokenSecretManager = yarnCluster.GetResourceManager
().GetRMContext().GetContainerTokenSecretManager();
Resource r = Resource.NewInstance(1230, 2);
Org.Apache.Hadoop.Yarn.Api.Records.Token containerToken = containerTokenSecretManager
.CreateContainerToken(cId, nodeId, user, r, Priority.NewInstance(0), 0);
ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
);
byte[] tokenIdentifierContent = ((byte[])containerToken.GetIdentifier().Array());
DataInputBuffer dib = new DataInputBuffer();
dib.Reset(tokenIdentifierContent, tokenIdentifierContent.Length);
containerTokenIdentifier.ReadFields(dib);
NUnit.Framework.Assert.AreEqual(cId, containerTokenIdentifier.GetContainerID());
NUnit.Framework.Assert.AreEqual(cId.ToString(), containerTokenIdentifier.GetContainerID
().ToString());
Org.Apache.Hadoop.Yarn.Api.Records.Token nmToken = nmTokenSecretManagerInRM.CreateNMToken
(appAttemptId, nodeId, user);
YarnRPC rpc = YarnRPC.Create(conf);
TestStartContainer(rpc, appAttemptId, nodeId, containerToken, nmToken, false);
IList <ContainerId> containerIds = new List <ContainerId>();
containerIds.AddItem(cId);
ContainerManagementProtocol proxy = GetContainerManagementProtocolProxy(rpc, nmToken
, nodeId, user);
GetContainerStatusesResponse res = proxy.GetContainerStatuses(GetContainerStatusesRequest
.NewInstance(containerIds));
NUnit.Framework.Assert.IsNotNull(res.GetContainerStatuses()[0]);
NUnit.Framework.Assert.AreEqual(cId, res.GetContainerStatuses()[0].GetContainerId
());
NUnit.Framework.Assert.AreEqual(cId.ToString(), res.GetContainerStatuses()[0].GetContainerId
().ToString());
}
private void ClearLimits(ContainerId containerId)
{
if (IsCpuWeightEnabled())
{
DeleteCgroup(PathForCgroup(ControllerCpu, containerId.ToString()));
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public StartContainersResponse StartContainers(StartContainersRequest requests)
{
StartContainerRequest request = requests.GetStartContainerRequests()[0];
Log.Info("Container started by MyContainerManager: " + request);
launched = true;
IDictionary <string, string> env = request.GetContainerLaunchContext().GetEnvironment
();
Token containerToken = request.GetContainerToken();
ContainerTokenIdentifier tokenId = null;
try
{
tokenId = BuilderUtils.NewContainerTokenIdentifier(containerToken);
}
catch (IOException e)
{
throw RPCUtil.GetRemoteException(e);
}
ContainerId containerId = tokenId.GetContainerID();
containerIdAtContainerManager = containerId.ToString();
attemptIdAtContainerManager = containerId.GetApplicationAttemptId().ToString();
nmHostAtContainerManager = tokenId.GetNmHostAddress();
submitTimeAtContainerManager = long.Parse(env[ApplicationConstants.AppSubmitTimeEnv
]);
maxAppAttempts = System.Convert.ToInt32(env[ApplicationConstants.MaxAppAttemptsEnv
]);
return(StartContainersResponse.NewInstance(new Dictionary <string, ByteBuffer>(),
new AList <ContainerId>(), new Dictionary <ContainerId, SerializedException>()));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestContainerLaunchNullImage()
{
string appSubmitter = "nobody";
string appId = "APP_ID";
string containerId = "CONTAINER_ID";
string testImage = string.Empty;
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>(Org.Mockito.Mockito.ReturnsDeepStubs);
ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(Org.Mockito.Mockito.ReturnsDeepStubs
);
ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
();
Dictionary <string, string> env = new Dictionary <string, string>();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
Org.Mockito.Mockito.When(cId.GetApplicationAttemptId().GetApplicationId().ToString
()).ThenReturn(appId);
Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId);
Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env);
env[YarnConfiguration.NmDockerContainerExecutorImageName] = testImage;
dockerContainerExecutor.GetConf().Set(YarnConfiguration.NmDockerContainerExecutorImageName
, testImage);
Path scriptPath = new Path("file:///bin/echo");
Path tokensPath = new Path("file:///dev/null");
Path pidFile = new Path(workDir, "pid.txt");
dockerContainerExecutor.ActivateContainer(cId, pidFile);
dockerContainerExecutor.LaunchContainer(container, scriptPath, tokensPath, appSubmitter
, appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs());
}
public AMAttemptInfo(AMInfo amInfo, string jobId, string user)
{
this.nodeHttpAddress = string.Empty;
this.nodeId = string.Empty;
string nmHost = amInfo.GetNodeManagerHost();
int nmHttpPort = amInfo.GetNodeManagerHttpPort();
int nmPort = amInfo.GetNodeManagerPort();
if (nmHost != null)
{
this.nodeHttpAddress = nmHost + ":" + nmHttpPort;
NodeId nodeId = NodeId.NewInstance(nmHost, nmPort);
this.nodeId = nodeId.ToString();
}
this.id = amInfo.GetAppAttemptId().GetAttemptId();
this.startTime = amInfo.GetStartTime();
this.containerId = string.Empty;
this.logsLink = string.Empty;
ContainerId containerId = amInfo.GetContainerId();
if (containerId != null)
{
this.containerId = containerId.ToString();
this.logsLink = StringHelper.Join(MRWebAppUtil.GetYARNWebappScheme() + nodeHttpAddress
, StringHelper.Ujoin("node", "containerlogs", this.containerId, user));
}
}
public virtual void TestContainerLaunch()
{
string appSubmitter = "nobody";
string cmd = LinuxContainerExecutor.Commands.LaunchContainer.GetValue().ToString(
);
string appId = "APP_ID";
string containerId = "CONTAINER_ID";
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>();
ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
();
Dictionary <string, string> env = new Dictionary <string, string>();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId);
Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env);
Path scriptPath = new Path("file:///bin/echo");
Path tokensPath = new Path("file:///dev/null");
Path workDir = new Path("/tmp");
Path pidFile = new Path(workDir, "pid.txt");
mockExec.ActivateContainer(cId, pidFile);
int ret = mockExec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter
, appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs());
NUnit.Framework.Assert.AreEqual(0, ret);
NUnit.Framework.Assert.AreEqual(Arrays.AsList(YarnConfiguration.DefaultNmNonsecureModeLocalUser
, appSubmitter, cmd, appId, containerId, workDir.ToString(), "/bin/echo", "/dev/null"
, pidFile.ToString(), StringUtils.Join(",", dirsHandler.GetLocalDirs()), StringUtils
.Join(",", dirsHandler.GetLogDirs()), "cgroups=none"), ReadMockParams());
}
public virtual void TestContainerIdWithEpoch()
{
ContainerId id = Org.Apache.Hadoop.Yarn.Api.TestContainerId.NewContainerId(0, 0,
0, 25645811);
string cid = ConverterUtils.ToString(id);
NUnit.Framework.Assert.AreEqual("container_0_0000_00_25645811", cid);
ContainerId gen = ConverterUtils.ToContainerId(cid);
NUnit.Framework.Assert.AreEqual(gen.ToString(), id.ToString());
long ts = Runtime.CurrentTimeMillis();
ContainerId id2 = Org.Apache.Hadoop.Yarn.Api.TestContainerId.NewContainerId(36473
, 4365472, ts, 4298334883325L);
string cid2 = ConverterUtils.ToString(id2);
NUnit.Framework.Assert.AreEqual("container_e03_" + ts + "_36473_4365472_999799999997"
, cid2);
ContainerId gen2 = ConverterUtils.ToContainerId(cid2);
NUnit.Framework.Assert.AreEqual(gen2.ToString(), id2.ToString());
ContainerId id3 = Org.Apache.Hadoop.Yarn.Api.TestContainerId.NewContainerId(36473
, 4365472, ts, 844424930131965L);
string cid3 = ConverterUtils.ToString(id3);
NUnit.Framework.Assert.AreEqual("container_e767_" + ts + "_36473_4365472_1099511627773"
, cid3);
ContainerId gen3 = ConverterUtils.ToContainerId(cid3);
NUnit.Framework.Assert.AreEqual(gen3.ToString(), id3.ToString());
}
/// <exception cref="System.IO.IOException"/>
public override void RemoveContainer(ContainerId containerId)
{
string keyPrefix = ContainersKeyPrefix + containerId.ToString();
try
{
WriteBatch batch = db.CreateWriteBatch();
try
{
batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerRequestKeySuffix));
batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerDiagsKeySuffix));
batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerLaunchedKeySuffix));
batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerKilledKeySuffix));
batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerExitCodeKeySuffix));
db.Write(batch);
}
finally
{
batch.Close();
}
}
catch (DBException e)
{
throw new IOException(e);
}
}
public AMAttemptInfo(AMInfo amInfo, string jobId, string user, string host, string
pathPrefix)
{
this.nodeHttpAddress = string.Empty;
this.nodeId = string.Empty;
string nmHost = amInfo.GetNodeManagerHost();
int nmHttpPort = amInfo.GetNodeManagerHttpPort();
int nmPort = amInfo.GetNodeManagerPort();
if (nmHost != null)
{
this.nodeHttpAddress = nmHost + ":" + nmHttpPort;
NodeId nodeId = NodeId.NewInstance(nmHost, nmPort);
this.nodeId = nodeId.ToString();
}
this.id = amInfo.GetAppAttemptId().GetAttemptId();
this.startTime = amInfo.GetStartTime();
this.containerId = string.Empty;
this.logsLink = string.Empty;
this.shortLogsLink = string.Empty;
ContainerId containerId = amInfo.GetContainerId();
if (containerId != null)
{
this.containerId = containerId.ToString();
this.logsLink = StringHelper.Join(host, pathPrefix, StringHelper.Ujoin("logs", this
.nodeId, this.containerId, jobId, user));
this.shortLogsLink = StringHelper.Ujoin("logs", this.nodeId, this.containerId, jobId
, user);
}
}
public virtual void TestForCorruptedAggregatedLogs()
{
Configuration conf = new Configuration();
FilePath workDir = new FilePath(testWorkDir, "testReadAcontainerLogs1");
Path remoteAppLogFile = new Path(workDir.GetAbsolutePath(), "aggregatedLogFile");
Path srcFileRoot = new Path(workDir.GetAbsolutePath(), "srcFiles");
ContainerId testContainerId = TestContainerId.NewContainerId(1, 1, 1, 1);
Path t = new Path(srcFileRoot, testContainerId.GetApplicationAttemptId().GetApplicationId
().ToString());
Path srcFilePath = new Path(t, testContainerId.ToString());
long numChars = 950000;
WriteSrcFileAndALog(srcFilePath, "stdout", numChars, remoteAppLogFile, srcFileRoot
, testContainerId);
AggregatedLogFormat.LogReader logReader = new AggregatedLogFormat.LogReader(conf,
remoteAppLogFile);
AggregatedLogFormat.LogKey rLogKey = new AggregatedLogFormat.LogKey();
DataInputStream dis = logReader.Next(rLogKey);
TextWriter writer = new StringWriter();
try
{
AggregatedLogFormat.LogReader.ReadAcontainerLogs(dis, writer);
}
catch (Exception e)
{
if (e.ToString().Contains("NumberFormatException"))
{
NUnit.Framework.Assert.Fail("Aggregated logs are corrupted.");
}
}
}
/// <summary>A helper api for creating an audit log for a failure event.</summary>
internal static string CreateFailureLog(string user, string operation, string perm
, string target, string description, ApplicationId appId, ApplicationAttemptId attemptId
, ContainerId containerId)
{
StringBuilder b = new StringBuilder();
Start(RMAuditLogger.Keys.User, user, b);
AddRemoteIP(b);
Add(RMAuditLogger.Keys.Operation, operation, b);
Add(RMAuditLogger.Keys.Target, target, b);
Add(RMAuditLogger.Keys.Result, RMAuditLogger.AuditConstants.Failure, b);
Add(RMAuditLogger.Keys.Description, description, b);
Add(RMAuditLogger.Keys.Permissions, perm, b);
if (appId != null)
{
Add(RMAuditLogger.Keys.Appid, appId.ToString(), b);
}
if (attemptId != null)
{
Add(RMAuditLogger.Keys.Appattemptid, attemptId.ToString(), b);
}
if (containerId != null)
{
Add(RMAuditLogger.Keys.Containerid, containerId.ToString(), b);
}
return(b.ToString());
}
private ContainerId GetNextContainerId()
{
ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>();
string id = "CONTAINER_" + GetNextId();
Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(id);
return(cId);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public static void StartContainer(NodeManager nm, ContainerId cId, FileContext localFS
, FilePath scriptFileDir, FilePath processStartFile)
{
FilePath scriptFile = CreateUnhaltingScriptFile(cId, scriptFileDir, processStartFile
);
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
NodeId nodeId = BuilderUtils.NewNodeId(Sharpen.Extensions.GetAddressByName("localhost"
).ToString(), 12345);
URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(scriptFile.GetAbsolutePath())));
LocalResource localResource = recordFactory.NewRecordInstance <LocalResource>();
localResource.SetResource(localResourceUri);
localResource.SetSize(-1);
localResource.SetVisibility(LocalResourceVisibility.Application);
localResource.SetType(LocalResourceType.File);
localResource.SetTimestamp(scriptFile.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = localResource;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile));
containerLaunchContext.SetCommands(commands);
IPEndPoint containerManagerBindAddress = NetUtils.CreateSocketAddrForHost("127.0.0.1"
, 12345);
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.ToString
());
Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> nmToken = ConverterUtils
.ConvertFromYarn(nm.GetNMContext().GetNMTokenSecretManager().CreateNMToken(cId.GetApplicationAttemptId
(), nodeId, user), containerManagerBindAddress);
currentUser.AddToken(nmToken);
ContainerManagementProtocol containerManager = currentUser.DoAs(new _PrivilegedAction_229
());
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, TestContainerManager.CreateContainerToken(cId, 0, nodeId, user, nm.GetNMContext
().GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds
);
ContainerStatus containerStatus = containerManager.GetContainerStatuses(request).
GetContainerStatuses()[0];
NUnit.Framework.Assert.AreEqual(ContainerState.Running, containerStatus.GetState(
));
}
/// <exception cref="System.Exception"/>
public virtual void TestInvalidContainerReleaseRequest()
{
MockRM rm = new MockRM(conf);
try
{
rm.Start();
// Register node1
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
// Submit an application
RMApp app1 = rm.SubmitApp(1024);
// kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId());
am1.RegisterAppAttempt();
am1.AddRequests(new string[] { "127.0.0.1" }, Gb, 1, 1);
AllocateResponse alloc1Response = am1.Schedule();
// send the request
// kick the scheduler
nm1.NodeHeartbeat(true);
while (alloc1Response.GetAllocatedContainers().Count < 1)
{
Log.Info("Waiting for containers to be created for app 1...");
Sharpen.Thread.Sleep(1000);
alloc1Response = am1.Schedule();
}
NUnit.Framework.Assert.IsTrue(alloc1Response.GetAllocatedContainers().Count > 0);
RMApp app2 = rm.SubmitApp(1024);
nm1.NodeHeartbeat(true);
RMAppAttempt attempt2 = app2.GetCurrentAppAttempt();
MockAM am2 = rm.SendAMLaunched(attempt2.GetAppAttemptId());
am2.RegisterAppAttempt();
// Now trying to release container allocated for app1 -> appAttempt1.
ContainerId cId = alloc1Response.GetAllocatedContainers()[0].GetId();
am2.AddContainerToBeReleased(cId);
try
{
am2.Schedule();
NUnit.Framework.Assert.Fail("Exception was expected!!");
}
catch (InvalidContainerReleaseException e)
{
StringBuilder sb = new StringBuilder("Cannot release container : ");
sb.Append(cId.ToString());
sb.Append(" not belonging to this application attempt : ");
sb.Append(attempt2.GetAppAttemptId().ToString());
NUnit.Framework.Assert.IsTrue(e.Message.Contains(sb.ToString()));
}
}
finally
{
if (rm != null)
{
rm.Stop();
}
}
}
/// <summary>Print all the information about this AM.</summary>
public virtual void PrintAll()
{
System.Console.Out.WriteLine("APPLICATION_ATTEMPT_ID:" + appAttemptId.ToString());
System.Console.Out.WriteLine("START_TIME: " + startTime);
System.Console.Out.WriteLine("CONTAINER_ID: " + containerId.ToString());
System.Console.Out.WriteLine("NODE_MANAGER_HOST: " + nodeManagerHost);
System.Console.Out.WriteLine("NODE_MANAGER_PORT: " + nodeManagerPort);
System.Console.Out.WriteLine("NODE_MANAGER_HTTP_PORT: " + nodeManagerHttpPort);
}
private ContainerId GetNextContainerId()
{
ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(Org.Mockito.Mockito.ReturnsDeepStubs
);
string id = "CONTAINER_" + Runtime.CurrentTimeMillis();
Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(id);
return(cId);
}
/// <exception cref="System.IO.IOException"/>
private void TestReadHistoryData(int num, bool missingContainer, bool missingApplicationAttempt
)
{
// read application history data
NUnit.Framework.Assert.AreEqual(num, store.GetAllApplications().Count);
for (int i = 1; i <= num; ++i)
{
ApplicationId appId = ApplicationId.NewInstance(0, i);
ApplicationHistoryData appData = store.GetApplication(appId);
NUnit.Framework.Assert.IsNotNull(appData);
NUnit.Framework.Assert.AreEqual(appId.ToString(), appData.GetApplicationName());
NUnit.Framework.Assert.AreEqual(appId.ToString(), appData.GetDiagnosticsInfo());
// read application attempt history data
NUnit.Framework.Assert.AreEqual(num, store.GetApplicationAttempts(appId).Count);
for (int j = 1; j <= num; ++j)
{
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, j);
ApplicationAttemptHistoryData attemptData = store.GetApplicationAttempt(appAttemptId
);
NUnit.Framework.Assert.IsNotNull(attemptData);
NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), attemptData.GetHost());
if (missingApplicationAttempt && j == num)
{
NUnit.Framework.Assert.IsNull(attemptData.GetDiagnosticsInfo());
continue;
}
else
{
NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), attemptData.GetDiagnosticsInfo
());
}
// read container history data
NUnit.Framework.Assert.AreEqual(num, store.GetContainers(appAttemptId).Count);
for (int k = 1; k <= num; ++k)
{
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, k);
ContainerHistoryData containerData = store.GetContainer(containerId);
NUnit.Framework.Assert.IsNotNull(containerData);
NUnit.Framework.Assert.AreEqual(Priority.NewInstance(containerId.GetId()), containerData
.GetPriority());
if (missingContainer && k == num)
{
NUnit.Framework.Assert.IsNull(containerData.GetDiagnosticsInfo());
}
else
{
NUnit.Framework.Assert.AreEqual(containerId.ToString(), containerData.GetDiagnosticsInfo
());
}
}
ContainerHistoryData masterContainer = store.GetAMContainer(appAttemptId);
NUnit.Framework.Assert.IsNotNull(masterContainer);
NUnit.Framework.Assert.AreEqual(ContainerId.NewContainerId(appAttemptId, 1), masterContainer
.GetContainerId());
}
}
}
public virtual void TestContainerLaunchError()
{
// reinitialize executer
FilePath f = new FilePath("./src/test/resources/mock-container-executer-with-error"
);
if (!FileUtil.CanExecute(f))
{
FileUtil.SetExecutable(f, true);
}
string executorPath = f.GetAbsolutePath();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, executorPath);
conf.Set(YarnConfiguration.NmLocalDirs, "file:///bin/echo");
conf.Set(YarnConfiguration.NmLogDirs, "file:///dev/null");
mockExec = Org.Mockito.Mockito.Spy(new LinuxContainerExecutor());
Org.Mockito.Mockito.DoAnswer(new _Answer_226()).When(mockExec).LogOutput(Matchers.Any
<string>());
dirsHandler = new LocalDirsHandlerService();
dirsHandler.Init(conf);
mockExec.SetConf(conf);
string appSubmitter = "nobody";
string cmd = LinuxContainerExecutor.Commands.LaunchContainer.GetValue().ToString(
);
string appId = "APP_ID";
string containerId = "CONTAINER_ID";
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>();
ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
();
Dictionary <string, string> env = new Dictionary <string, string>();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
Org.Mockito.Mockito.DoAnswer(new _Answer_254()).When(container).Handle(Matchers.Any
<ContainerDiagnosticsUpdateEvent>());
Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId);
Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env);
Path scriptPath = new Path("file:///bin/echo");
Path tokensPath = new Path("file:///dev/null");
Path workDir = new Path("/tmp");
Path pidFile = new Path(workDir, "pid.txt");
mockExec.ActivateContainer(cId, pidFile);
int ret = mockExec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter
, appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs());
NUnit.Framework.Assert.AreNotSame(0, ret);
NUnit.Framework.Assert.AreEqual(Arrays.AsList(YarnConfiguration.DefaultNmNonsecureModeLocalUser
, appSubmitter, cmd, appId, containerId, workDir.ToString(), "/bin/echo", "/dev/null"
, pidFile.ToString(), StringUtils.Join(",", dirsHandler.GetLocalDirs()), StringUtils
.Join(",", dirsHandler.GetLogDirs()), "cgroups=none"), ReadMockParams());
}
/// <exception cref="System.IO.IOException"/>
public virtual ContainerHistoryData GetContainer(ContainerId containerId)
{
FileSystemApplicationHistoryStore.HistoryFileReader hfReader = GetHistoryFileReader
(containerId.GetApplicationAttemptId().GetApplicationId());
try
{
bool readStartData = false;
bool readFinishData = false;
ContainerHistoryData historyData = ContainerHistoryData.NewInstance(containerId,
null, null, null, long.MinValue, long.MaxValue, null, int.MaxValue, null);
while ((!readStartData || !readFinishData) && hfReader.HasNext())
{
FileSystemApplicationHistoryStore.HistoryFileReader.Entry entry = hfReader.Next();
if (entry.key.id.Equals(containerId.ToString()))
{
if (entry.key.suffix.Equals(StartDataSuffix))
{
ContainerStartData startData = ParseContainerStartData(entry.value);
MergeContainerHistoryData(historyData, startData);
readStartData = true;
}
else
{
if (entry.key.suffix.Equals(FinishDataSuffix))
{
ContainerFinishData finishData = ParseContainerFinishData(entry.value);
MergeContainerHistoryData(historyData, finishData);
readFinishData = true;
}
}
}
}
if (!readStartData && !readFinishData)
{
return(null);
}
if (!readStartData)
{
Log.Warn("Start information is missing for container " + containerId);
}
if (!readFinishData)
{
Log.Warn("Finish information is missing for container " + containerId);
}
Log.Info("Completed reading history information of container " + containerId);
return(historyData);
}
catch (IOException e)
{
Log.Error("Error when reading history file of container " + containerId, e);
throw;
}
finally
{
hfReader.Close();
}
}
private static TimelineEntity CreateContainerEntity(ContainerId containerId)
{
TimelineEntity entity = new TimelineEntity();
entity.SetEntityType(ContainerMetricsConstants.EntityType);
entity.SetEntityId(containerId.ToString());
entity.AddPrimaryFilter(ContainerMetricsConstants.ParentPrimariyFilter, containerId
.GetApplicationAttemptId().ToString());
return(entity);
}
/// <summary>Create an event to record the start of an MR AppMaster</summary>
/// <param name="appAttemptId">the application attempt id.</param>
/// <param name="startTime">the start time of the AM.</param>
/// <param name="containerId">the containerId of the AM.</param>
/// <param name="nodeManagerHost">the node on which the AM is running.</param>
/// <param name="nodeManagerPort">the port on which the AM is running.</param>
/// <param name="nodeManagerHttpPort">the httpPort for the node running the AM.</param>
/// <param name="forcedJobStateOnShutDown">the state to force the job into</param>
public AMStartedEvent(ApplicationAttemptId appAttemptId, long startTime, ContainerId
containerId, string nodeManagerHost, int nodeManagerPort, int nodeManagerHttpPort
, string forcedJobStateOnShutDown, long submitTime)
{
datum.applicationAttemptId = new Utf8(appAttemptId.ToString());
datum.startTime = startTime;
datum.containerId = new Utf8(containerId.ToString());
datum.nodeManagerHost = new Utf8(nodeManagerHost);
datum.nodeManagerPort = nodeManagerPort;
datum.nodeManagerHttpPort = nodeManagerHttpPort;
this.forcedJobStateOnShutDown = forcedJobStateOnShutDown;
this.submitTime = submitTime;
}
/// <exception cref="System.IO.IOException"/>
public override void StoreContainerKilled(ContainerId containerId)
{
string key = ContainersKeyPrefix + containerId.ToString() + ContainerKilledKeySuffix;
try
{
db.Put(JniDBFactory.Bytes(key), EmptyValue);
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
public override void StoreContainerDiagnostics(ContainerId containerId, StringBuilder
diagnostics)
{
string key = ContainersKeyPrefix + containerId.ToString() + ContainerDiagsKeySuffix;
try
{
db.Put(JniDBFactory.Bytes(key), JniDBFactory.Bytes(diagnostics.ToString()));
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual void TestContainerLogFile()
{
FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir"
).GetAbsoluteFile();
string logdirwithFile = absLogDir.ToURI().ToString();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile);
conf.SetFloat(YarnConfiguration.NmMaxPerDiskUtilizationPercentage, 0.0f);
LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
dirsHandler.Init(conf);
NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler
, new ApplicationACLsManager(conf), new NMNullStateStoreService());
// Add an application and the corresponding containers
string user = "******";
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(clusterTimeStamp, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1);
nmContext.GetApplications()[appId] = app;
MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
conf, user, appId, 1);
container.SetState(ContainerState.Running);
nmContext.GetContainers()[containerId] = container;
FilePath containerLogDir = new FilePath(absLogDir, ContainerLaunch.GetRelativeContainerLogDir
(appId.ToString(), containerId.ToString()));
containerLogDir.Mkdirs();
string fileName = "fileName";
FilePath containerLogFile = new FilePath(containerLogDir, fileName);
containerLogFile.CreateNewFile();
FilePath file = ContainerLogsUtils.GetContainerLogFile(containerId, fileName, user
, nmContext);
NUnit.Framework.Assert.AreEqual(containerLogFile.ToURI().ToString(), file.ToURI()
.ToString());
FileUtil.FullyDelete(absLogDir);
}
/// <exception cref="System.IO.IOException"/>
public override void StoreContainerCompleted(ContainerId containerId, int exitCode
)
{
string key = ContainersKeyPrefix + containerId.ToString() + ContainerExitCodeKeySuffix;
try
{
db.Put(JniDBFactory.Bytes(key), JniDBFactory.Bytes(Sharpen.Extensions.ToString(exitCode
)));
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
public override void StoreContainer(ContainerId containerId, StartContainerRequest
startRequest)
{
string key = ContainersKeyPrefix + containerId.ToString() + ContainerRequestKeySuffix;
try
{
db.Put(JniDBFactory.Bytes(key), ((StartContainerRequestPBImpl)startRequest).GetProto
().ToByteArray());
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.Exception"/>
private static void CreateContainerLogInLocalDir(Path appLogsDir, ContainerId containerId
, FileSystem fs)
{
Path containerLogsDir = new Path(appLogsDir, containerId.ToString());
if (fs.Exists(containerLogsDir))
{
fs.Delete(containerLogsDir, true);
}
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(containerLogsDir));
TextWriter writer = new FileWriter(new FilePath(containerLogsDir.ToString(), "sysout"
));
writer.Write("Hello " + containerId + "!");
writer.Close();
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
internal static IList <FilePath> GetContainerLogDirs(ContainerId containerId, LocalDirsHandlerService
dirsHandler)
{
IList <string> logDirs = dirsHandler.GetLogDirsForRead();
IList <FilePath> containerLogDirs = new AList <FilePath>(logDirs.Count);
foreach (string logDir in logDirs)
{
logDir = new FilePath(logDir).ToURI().GetPath();
string appIdStr = ConverterUtils.ToString(containerId.GetApplicationAttemptId().GetApplicationId
());
FilePath appLogDir = new FilePath(logDir, appIdStr);
containerLogDirs.AddItem(new FilePath(appLogDir, containerId.ToString()));
}
return(containerLogDirs);
}
public virtual string GetResourcesOption(ContainerId containerId)
{
string containerName = containerId.ToString();
StringBuilder sb = new StringBuilder("cgroups=");
if (IsCpuWeightEnabled())
{
sb.Append(PathForCgroup(ControllerCpu, containerName) + "/tasks");
sb.Append(",");
}
if (sb[sb.Length - 1] == ',')
{
Sharpen.Runtime.DeleteCharAt(sb, sb.Length - 1);
}
return(sb.ToString());
}
public virtual void TestLogFileWithDriveLetter()
{
ContainerImpl container = Org.Mockito.Mockito.Mock <ContainerImpl>();
ApplicationIdPBImpl appId = Org.Mockito.Mockito.Mock <ApplicationIdPBImpl>();
Org.Mockito.Mockito.When(appId.ToString()).ThenReturn("appId");
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
ApplicationAttemptIdPBImpl appAttemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptIdPBImpl
>();
Org.Mockito.Mockito.When(appAttemptId.GetApplicationId()).ThenReturn(appId);
ConcurrentMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
> applications = new ConcurrentHashMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
applications[appId] = app;
ContainerId containerId = Org.Mockito.Mockito.Mock <ContainerIdPBImpl>();
Org.Mockito.Mockito.When(containerId.ToString()).ThenReturn("containerId");
Org.Mockito.Mockito.When(containerId.GetApplicationAttemptId()).ThenReturn(appAttemptId
);
ConcurrentMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
> containers = new ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
containers[containerId] = container;
LocalDirsHandlerService localDirs = Org.Mockito.Mockito.Mock <LocalDirsHandlerService
>();
Org.Mockito.Mockito.When(localDirs.GetLogPathToRead("appId" + Path.Separator + "containerId"
+ Path.Separator + "fileName")).ThenReturn(new Path("F:/nmlogs/appId/containerId/fileName"
));
NodeManager.NMContext context = Org.Mockito.Mockito.Mock <NodeManager.NMContext>();
Org.Mockito.Mockito.When(context.GetLocalDirsHandler()).ThenReturn(localDirs);
Org.Mockito.Mockito.When(context.GetApplications()).ThenReturn(applications);
Org.Mockito.Mockito.When(context.GetContainers()).ThenReturn(containers);
FilePath logFile = ContainerLogsUtils.GetContainerLogFile(containerId, "fileName"
, null, context);
NUnit.Framework.Assert.IsTrue("logFile lost drive letter " + logFile, logFile.ToString
().IndexOf("F:" + FilePath.separator + "nmlogs") > -1);
}
