/// <summary>Finds the log file with the given filename for the given container.</summary>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public static FilePath GetContainerLogFile(ContainerId containerId, string fileName
, string remoteUser, Context context)
{
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= context.GetContainers()[containerId];
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
application = GetApplicationForContainer(containerId, context);
CheckAccess(remoteUser, application, context);
if (container != null)
{
CheckState(container.GetContainerState());
}
try
{
LocalDirsHandlerService dirsHandler = context.GetLocalDirsHandler();
string relativeContainerLogDir = ContainerLaunch.GetRelativeContainerLogDir(application
.GetAppId().ToString(), containerId.ToString());
Path logPath = dirsHandler.GetLogPathToRead(relativeContainerLogDir + Path.Separator
+ fileName);
URI logPathURI = new FilePath(logPath.ToString()).ToURI();
FilePath logFile = new FilePath(logPathURI.GetPath());
return(logFile);
}
catch (IOException e)
{
Log.Warn("Failed to find log file", e);
throw new NotFoundException("Cannot find this log on the local disk.");
}
}
public virtual void TestContainerLogs()
{
WebResource r = Resource();
ContainerId containerId = BuilderUtils.NewContainerId(0, 0, 0, 0);
string containerIdStr = BuilderUtils.NewContainerId(0, 0, 0, 0).ToString();
ApplicationAttemptId appAttemptId = containerId.GetApplicationAttemptId();
ApplicationId appId = appAttemptId.GetApplicationId();
string appIdStr = appId.ToString();
string filename = "logfile1";
string logMessage = "log message\n";
nmContext.GetApplications()[appId] = new ApplicationImpl(null, "user", appId, null
, nmContext);
MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
new Configuration(), "user", appId, 1);
container.SetState(ContainerState.Running);
nmContext.GetContainers()[containerId] = container;
// write out log file
Path path = dirsHandler.GetLogPathForWrite(ContainerLaunch.GetRelativeContainerLogDir
(appIdStr, containerIdStr) + "/" + filename, false);
FilePath logFile = new FilePath(path.ToUri().GetPath());
logFile.DeleteOnExit();
NUnit.Framework.Assert.IsTrue("Failed to create log dir", logFile.GetParentFile()
.Mkdirs());
PrintWriter pw = new PrintWriter(logFile);
pw.Write(logMessage);
pw.Close();
// ask for it
ClientResponse response = r.Path("ws").Path("v1").Path("node").Path("containerlogs"
).Path(containerIdStr).Path(filename).Accept(MediaType.TextPlain).Get <ClientResponse
>();
string responseText = response.GetEntity <string>();
NUnit.Framework.Assert.AreEqual(logMessage, responseText);
// ask for file that doesn't exist
response = r.Path("ws").Path("v1").Path("node").Path("containerlogs").Path(containerIdStr
).Path("uhhh").Accept(MediaType.TextPlain).Get <ClientResponse>();
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.NotFound.GetStatusCode(), response
.GetStatus());
responseText = response.GetEntity <string>();
NUnit.Framework.Assert.IsTrue(responseText.Contains("Cannot find this log on the local disk."
));
// After container is completed, it is removed from nmContext
Sharpen.Collections.Remove(nmContext.GetContainers(), containerId);
NUnit.Framework.Assert.IsNull(nmContext.GetContainers()[containerId]);
response = r.Path("ws").Path("v1").Path("node").Path("containerlogs").Path(containerIdStr
).Path(filename).Accept(MediaType.TextPlain).Get <ClientResponse>();
responseText = response.GetEntity <string>();
NUnit.Framework.Assert.AreEqual(logMessage, responseText);
}
protected internal override void WriteLocalWrapperScript(Path launchDst, Path pidFile
, TextWriter pout)
{
string exitCodeFile = ContainerLaunch.GetExitCodeFile(pidFile.ToString());
string tmpFile = exitCodeFile + ".tmp";
pout.WriteLine("#!/usr/bin/env bash");
pout.WriteLine("bash \"" + this.sessionScriptPath.ToString() + "\"");
pout.WriteLine("rc=$?");
pout.WriteLine("echo $rc > \"" + tmpFile + "\"");
pout.WriteLine("mv -f \"" + tmpFile + "\" \"" + exitCodeFile + "\"");
pout.WriteLine("exit $rc");
}
/// <summary>Recover an already existing container.</summary>
/// <remarks>
/// Recover an already existing container. This is a blocking call and returns
/// only when the container exits. Note that the container must have been
/// activated prior to this call.
/// </remarks>
/// <param name="user">the user of the container</param>
/// <param name="containerId">The ID of the container to reacquire</param>
/// <returns>The exit code of the pre-existing container</returns>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"></exception>
public virtual int ReacquireContainer(string user, ContainerId containerId)
{
Path pidPath = GetPidFilePath(containerId);
if (pidPath == null)
{
Log.Warn(containerId + " is not active, returning terminated error");
return(ContainerExecutor.ExitCode.Terminated.GetExitCode());
}
string pid = null;
pid = ProcessIdFileReader.GetProcessId(pidPath);
if (pid == null)
{
throw new IOException("Unable to determine pid for " + containerId);
}
Log.Info("Reacquiring " + containerId + " with pid " + pid);
while (IsContainerProcessAlive(user, pid))
{
Sharpen.Thread.Sleep(1000);
}
// wait for exit code file to appear
string exitCodeFile = ContainerLaunch.GetExitCodeFile(pidPath.ToString());
FilePath file = new FilePath(exitCodeFile);
int sleepMsec = 100;
int msecLeft = 2000;
while (!file.Exists() && msecLeft >= 0)
{
if (!IsContainerActive(containerId))
{
Log.Info(containerId + " was deactivated");
return(ContainerExecutor.ExitCode.Terminated.GetExitCode());
}
Sharpen.Thread.Sleep(sleepMsec);
msecLeft -= sleepMsec;
}
if (msecLeft < 0)
{
throw new IOException("Timeout while waiting for exit code from " + containerId);
}
try
{
return(System.Convert.ToInt32(FileUtils.ReadFileToString(file).Trim()));
}
catch (FormatException e)
{
throw new IOException("Error parsing exit code from pid " + pid, e);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual void TestContainerLogFile()
{
FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir"
).GetAbsoluteFile();
string logdirwithFile = absLogDir.ToURI().ToString();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile);
conf.SetFloat(YarnConfiguration.NmMaxPerDiskUtilizationPercentage, 0.0f);
LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
dirsHandler.Init(conf);
NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler
, new ApplicationACLsManager(conf), new NMNullStateStoreService());
// Add an application and the corresponding containers
string user = "******";
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(clusterTimeStamp, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1);
nmContext.GetApplications()[appId] = app;
MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
conf, user, appId, 1);
container.SetState(ContainerState.Running);
nmContext.GetContainers()[containerId] = container;
FilePath containerLogDir = new FilePath(absLogDir, ContainerLaunch.GetRelativeContainerLogDir
(appId.ToString(), containerId.ToString()));
containerLogDir.Mkdirs();
string fileName = "fileName";
FilePath containerLogFile = new FilePath(containerLogDir, fileName);
containerLogFile.CreateNewFile();
FilePath file = ContainerLogsUtils.GetContainerLogFile(containerId, fileName, user
, nmContext);
NUnit.Framework.Assert.AreEqual(containerLogFile.ToURI().ToString(), file.ToURI()
.ToString());
FileUtil.FullyDelete(absLogDir);
}