protected internal LocalWrapperScriptBuilder(DefaultContainerExecutor _enclosing,
Path containerWorkDir)
{
this._enclosing = _enclosing;
this.wrapperScriptPath = new Path(containerWorkDir, Shell.AppendScriptExtension("default_container_executor"
));
}
public WindowsLocalWrapperScriptBuilder(DefaultContainerExecutor _enclosing, string
containerIdStr, Path containerWorkDir)
: base(_enclosing)
{
this._enclosing = _enclosing;
this.containerIdStr = containerIdStr;
}
public UnixLocalWrapperScriptBuilder(DefaultContainerExecutor _enclosing, Path containerWorkDir
)
: base(_enclosing)
{
this._enclosing = _enclosing;
this.sessionScriptPath = new Path(containerWorkDir, Shell.AppendScriptExtension("default_container_executor_session"
));
}
public virtual void TestKillContainersOnShutdown()
{
nm = new TestNodeManagerShutdown.TestNodeManager(this);
nm.Init(CreateNMConfig());
nm.Start();
StartContainer(nm, cId, localFS, tmpDir, processStartFile);
int MaxTries = 20;
int numTries = 0;
while (!processStartFile.Exists() && numTries < MaxTries)
{
try
{
Sharpen.Thread.Sleep(500);
}
catch (Exception ex)
{
Sharpen.Runtime.PrintStackTrace(ex);
}
numTries++;
}
nm.Stop();
// Now verify the contents of the file. Script generates a message when it
// receives a sigterm so we look for that. We cannot perform this check on
// Windows, because the process is not notified when killed by winutils.
// There is no way for the process to trap and respond. Instead, we can
// verify that the job object with ID matching container ID no longer exists.
if (Shell.Windows)
{
NUnit.Framework.Assert.IsFalse("Process is still alive!", DefaultContainerExecutor
.ContainerIsAlive(cId.ToString()));
}
else
{
BufferedReader reader = new BufferedReader(new FileReader(processStartFile));
bool foundSigTermMessage = false;
while (true)
{
string line = reader.ReadLine();
if (line == null)
{
break;
}
if (line.Contains("SIGTERM"))
{
foundSigTermMessage = true;
break;
}
}
NUnit.Framework.Assert.IsTrue("Did not find sigterm message", foundSigTermMessage
);
reader.Close();
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestStartLocalizer()
{
IPEndPoint localizationServerAddress;
Path firstDir = new Path(BaseTmpPath, "localDir1");
IList <string> localDirs = new AList <string>();
Path secondDir = new Path(BaseTmpPath, "localDir2");
IList <string> logDirs = new AList <string>();
Path logDir = new Path(BaseTmpPath, "logDir");
Path tokenDir = new Path(BaseTmpPath, "tokenDir");
FsPermission perms = new FsPermission((short)0x1f8);
Configuration conf = new Configuration();
localizationServerAddress = conf.GetSocketAddr(YarnConfiguration.NmBindHost, YarnConfiguration
.NmLocalizerAddress, YarnConfiguration.DefaultNmLocalizerAddress, YarnConfiguration
.DefaultNmLocalizerPort);
FileContext mockLfs = Org.Mockito.Mockito.Spy(FileContext.GetLocalFSFileContext(conf
));
FileContext.Util mockUtil = Org.Mockito.Mockito.Spy(mockLfs.Util());
Org.Mockito.Mockito.DoAnswer(new _Answer_344(mockUtil)).When(mockLfs).Util();
Org.Mockito.Mockito.DoAnswer(new _Answer_351(firstDir, mockLfs)).When(mockUtil).Copy
(Matchers.Any <Path>(), Matchers.Any <Path>());
// throw an Exception when copy token to the first local dir
// to simulate no space on the first drive
// copy token to the second local dir
Org.Mockito.Mockito.DoAnswer(new _Answer_378(firstDir)).When(mockLfs).GetFsStatus
(Matchers.Any <Path>());
// let second local directory return more free space than
// first local directory
DefaultContainerExecutor mockExec = Org.Mockito.Mockito.Spy(new DefaultContainerExecutor
(mockLfs));
mockExec.SetConf(conf);
localDirs.AddItem(mockLfs.MakeQualified(firstDir).ToString());
localDirs.AddItem(mockLfs.MakeQualified(secondDir).ToString());
logDirs.AddItem(mockLfs.MakeQualified(logDir).ToString());
conf.SetStrings(YarnConfiguration.NmLocalDirs, Sharpen.Collections.ToArray(localDirs
, new string[localDirs.Count]));
conf.Set(YarnConfiguration.NmLogDirs, logDir.ToString());
mockLfs.Mkdir(tokenDir, perms, true);
Path nmPrivateCTokensPath = new Path(tokenDir, "test.tokens");
string appSubmitter = "nobody";
string appId = "APP_ID";
string locId = "LOC_ID";
LocalDirsHandlerService dirsHandler = Org.Mockito.Mockito.Mock <LocalDirsHandlerService
>();
Org.Mockito.Mockito.When(dirsHandler.GetLocalDirs()).ThenReturn(localDirs);
Org.Mockito.Mockito.When(dirsHandler.GetLogDirs()).ThenReturn(logDirs);
try
{
mockExec.StartLocalizer(nmPrivateCTokensPath, localizationServerAddress, appSubmitter
, appId, locId, dirsHandler);
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail("StartLocalizer failed to copy token file " + e);
}
finally
{
mockExec.DeleteAsUser(appSubmitter, firstDir);
mockExec.DeleteAsUser(appSubmitter, secondDir);
mockExec.DeleteAsUser(appSubmitter, logDir);
DeleteTmpFiles();
}
}
public virtual void TestContainerLaunchError()
{
if (Shell.Windows)
{
BaseTmpPath = new Path(new FilePath("target").GetAbsolutePath(), typeof(TestDefaultContainerExecutor
).Name);
}
Path localDir = new Path(BaseTmpPath, "localDir");
IList <string> localDirs = new AList <string>();
localDirs.AddItem(localDir.ToString());
IList <string> logDirs = new AList <string>();
Path logDir = new Path(BaseTmpPath, "logDir");
logDirs.AddItem(logDir.ToString());
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077");
conf.Set(YarnConfiguration.NmLocalDirs, localDir.ToString());
conf.Set(YarnConfiguration.NmLogDirs, logDir.ToString());
FileContext lfs = FileContext.GetLocalFSFileContext(conf);
DefaultContainerExecutor mockExec = Org.Mockito.Mockito.Spy(new DefaultContainerExecutor
(lfs));
mockExec.SetConf(conf);
Org.Mockito.Mockito.DoAnswer(new _Answer_245()).When(mockExec).LogOutput(Matchers.Any
<string>());
string appSubmitter = "nobody";
string appId = "APP_ID";
string containerId = "CONTAINER_ID";
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>();
ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
();
Dictionary <string, string> env = new Dictionary <string, string>();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
try
{
Org.Mockito.Mockito.DoAnswer(new _Answer_268()).When(container).Handle(Matchers.Any
<ContainerDiagnosticsUpdateEvent>());
Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId);
Org.Mockito.Mockito.When(cId.GetApplicationAttemptId()).ThenReturn(ApplicationAttemptId
.NewInstance(ApplicationId.NewInstance(0, 1), 0));
Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env);
mockExec.CreateUserLocalDirs(localDirs, appSubmitter);
mockExec.CreateUserCacheDirs(localDirs, appSubmitter);
mockExec.CreateAppDirs(localDirs, appSubmitter, appId);
mockExec.CreateAppLogDirs(appId, logDirs, appSubmitter);
Path scriptPath = new Path("file:///bin/echo");
Path tokensPath = new Path("file:///dev/null");
if (Shell.Windows)
{
FilePath tmp = new FilePath(BaseTmpPath.ToString(), "test_echo.cmd");
BufferedWriter output = new BufferedWriter(new FileWriter(tmp));
output.Write("Exit 1");
output.Write("Echo No such file or directory 1>&2");
output.Close();
scriptPath = new Path(tmp.GetAbsolutePath());
tmp = new FilePath(BaseTmpPath.ToString(), "tokens");
tmp.CreateNewFile();
tokensPath = new Path(tmp.GetAbsolutePath());
}
Path workDir = localDir;
Path pidFile = new Path(workDir, "pid.txt");
mockExec.Init();
mockExec.ActivateContainer(cId, pidFile);
int ret = mockExec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter
, appId, workDir, localDirs, localDirs);
NUnit.Framework.Assert.AreNotSame(0, ret);
}
finally
{
mockExec.DeleteAsUser(appSubmitter, localDir);
mockExec.DeleteAsUser(appSubmitter, logDir);
}
}
public virtual void TestDirPermissions()
{
DeleteTmpFiles();
string user = "******";
string appId = "app_12345_123";
FsPermission userCachePerm = new FsPermission(DefaultContainerExecutor.UserPerm);
FsPermission appCachePerm = new FsPermission(DefaultContainerExecutor.AppcachePerm
);
FsPermission fileCachePerm = new FsPermission(DefaultContainerExecutor.FilecachePerm
);
FsPermission appDirPerm = new FsPermission(DefaultContainerExecutor.AppdirPerm);
FsPermission logDirPerm = new FsPermission(DefaultContainerExecutor.LogdirPerm);
IList <string> localDirs = new AList <string>();
localDirs.AddItem(new Path(BaseTmpPath, "localDirA").ToString());
localDirs.AddItem(new Path(BaseTmpPath, "localDirB").ToString());
IList <string> logDirs = new AList <string>();
logDirs.AddItem(new Path(BaseTmpPath, "logDirA").ToString());
logDirs.AddItem(new Path(BaseTmpPath, "logDirB").ToString());
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077");
FileContext lfs = FileContext.GetLocalFSFileContext(conf);
DefaultContainerExecutor executor = new DefaultContainerExecutor(lfs);
executor.Init();
try
{
executor.CreateUserLocalDirs(localDirs, user);
executor.CreateUserCacheDirs(localDirs, user);
executor.CreateAppDirs(localDirs, user, appId);
foreach (string dir in localDirs)
{
FileStatus stats = lfs.GetFileStatus(new Path(new Path(dir, ContainerLocalizer.Usercache
), user));
NUnit.Framework.Assert.AreEqual(userCachePerm, stats.GetPermission());
}
foreach (string dir_1 in localDirs)
{
Path userCachePath = new Path(new Path(dir_1, ContainerLocalizer.Usercache), user
);
Path appCachePath = new Path(userCachePath, ContainerLocalizer.Appcache);
FileStatus stats = lfs.GetFileStatus(appCachePath);
NUnit.Framework.Assert.AreEqual(appCachePerm, stats.GetPermission());
stats = lfs.GetFileStatus(new Path(userCachePath, ContainerLocalizer.Filecache));
NUnit.Framework.Assert.AreEqual(fileCachePerm, stats.GetPermission());
stats = lfs.GetFileStatus(new Path(appCachePath, appId));
NUnit.Framework.Assert.AreEqual(appDirPerm, stats.GetPermission());
}
executor.CreateAppLogDirs(appId, logDirs, user);
foreach (string dir_2 in logDirs)
{
FileStatus stats = lfs.GetFileStatus(new Path(dir_2, appId));
NUnit.Framework.Assert.AreEqual(logDirPerm, stats.GetPermission());
}
}
finally
{
DeleteTmpFiles();
}
}
public virtual void TestSuccessfulContainerLaunch()
{
FileContext localFS = FileContext.GetLocalFSFileContext();
localFS.Delete(new Path(localDir.GetAbsolutePath()), true);
localFS.Delete(new Path(localLogDir.GetAbsolutePath()), true);
localFS.Delete(new Path(remoteLogDir.GetAbsolutePath()), true);
localDir.Mkdir();
localLogDir.Mkdir();
remoteLogDir.Mkdir();
YarnConfiguration conf = new YarnConfiguration();
Context context = new _NMContext_84(new NMContainerTokenSecretManager(conf), new
NMTokenSecretManagerInNM(), null, null, new NMNullStateStoreService());
conf.Set(YarnConfiguration.NmLocalDirs, localDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmLogDirs, localLogDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmRemoteAppLogDir, remoteLogDir.GetAbsolutePath());
ContainerExecutor exec = new DefaultContainerExecutor();
exec.SetConf(conf);
DeletionService del = new DeletionService(exec);
Dispatcher dispatcher = new AsyncDispatcher();
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
NodeManagerMetrics metrics = NodeManagerMetrics.Create();
NodeStatusUpdater nodeStatusUpdater = new _NodeStatusUpdaterImpl_106(context, dispatcher
, healthChecker, metrics);
// Don't start any updating thread.
DummyContainerManager containerManager = new DummyContainerManager(context, exec,
del, nodeStatusUpdater, metrics, new ApplicationACLsManager(conf), dirsHandler);
nodeStatusUpdater.Init(conf);
((NodeManager.NMContext)context).SetContainerManager(containerManager);
nodeStatusUpdater.Start();
containerManager.Init(conf);
containerManager.Start();
ContainerLaunchContext launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
ApplicationId applicationId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 0);
ContainerId cID = ContainerId.NewContainerId(applicationAttemptId, 0);
string user = "******";
StartContainerRequest scRequest = StartContainerRequest.NewInstance(launchContext
, TestContainerManager.CreateContainerToken(cID, SimulatedRmIdentifier, context.
GetNodeId(), user, context.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
BaseContainerManagerTest.WaitForContainerState(containerManager, cID, ContainerState
.Running);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cID);
StopContainersRequest stopRequest = StopContainersRequest.NewInstance(containerIds
);
containerManager.StopContainers(stopRequest);
BaseContainerManagerTest.WaitForContainerState(containerManager, cID, ContainerState
.Complete);
containerManager.Stop();
}