/// <summary>Read the next key and return the value-stream.</summary>
/// <param name="key"/>
/// <returns>the valueStream if there are more keys or null otherwise.</returns>
/// <exception cref="System.IO.IOException"/>
public virtual DataInputStream Next(AggregatedLogFormat.LogKey key)
{
if (!this.atBeginning)
{
this.scanner.Advance();
}
else
{
this.atBeginning = false;
}
if (this.scanner.AtEnd())
{
return(null);
}
TFile.Reader.Scanner.Entry entry = this.scanner.Entry();
key.ReadFields(entry.GetKeyStream());
// Skip META keys
if (ReservedKeys.Contains(key.ToString()))
{
return(Next(key));
}
DataInputStream valueStream = entry.GetValueStream();
return(valueStream);
}
public virtual void TestForCorruptedAggregatedLogs()
{
Configuration conf = new Configuration();
FilePath workDir = new FilePath(testWorkDir, "testReadAcontainerLogs1");
Path remoteAppLogFile = new Path(workDir.GetAbsolutePath(), "aggregatedLogFile");
Path srcFileRoot = new Path(workDir.GetAbsolutePath(), "srcFiles");
ContainerId testContainerId = TestContainerId.NewContainerId(1, 1, 1, 1);
Path t = new Path(srcFileRoot, testContainerId.GetApplicationAttemptId().GetApplicationId
().ToString());
Path srcFilePath = new Path(t, testContainerId.ToString());
long numChars = 950000;
WriteSrcFileAndALog(srcFilePath, "stdout", numChars, remoteAppLogFile, srcFileRoot
, testContainerId);
AggregatedLogFormat.LogReader logReader = new AggregatedLogFormat.LogReader(conf,
remoteAppLogFile);
AggregatedLogFormat.LogKey rLogKey = new AggregatedLogFormat.LogKey();
DataInputStream dis = logReader.Next(rLogKey);
TextWriter writer = new StringWriter();
try
{
AggregatedLogFormat.LogReader.ReadAcontainerLogs(dis, writer);
}
catch (Exception e)
{
if (e.ToString().Contains("NumberFormatException"))
{
NUnit.Framework.Assert.Fail("Aggregated logs are corrupted.");
}
}
}
public override bool Equals(object obj)
{
if (obj is AggregatedLogFormat.LogKey)
{
AggregatedLogFormat.LogKey other = (AggregatedLogFormat.LogKey)obj;
if (this.keyString == null)
{
return(other.keyString == null);
}
return(this.keyString.Equals(other.keyString));
}
return(false);
}
/// <summary>Returns the owner of the application.</summary>
/// <returns>the application owner.</returns>
/// <exception cref="System.IO.IOException"/>
public virtual string GetApplicationOwner()
{
TFile.Reader.Scanner ownerScanner = reader.CreateScanner();
AggregatedLogFormat.LogKey key = new AggregatedLogFormat.LogKey();
while (!ownerScanner.AtEnd())
{
TFile.Reader.Scanner.Entry entry = ownerScanner.Entry();
key.ReadFields(entry.GetKeyStream());
if (key.ToString().Equals(ApplicationOwnerKey.ToString()))
{
DataInputStream valueStream = entry.GetValueStream();
return(valueStream.ReadUTF());
}
ownerScanner.Advance();
}
return(null);
}
/// <exception cref="System.IO.IOException"/>
public virtual void Append(AggregatedLogFormat.LogKey logKey, AggregatedLogFormat.LogValue
logValue)
{
ICollection <FilePath> pendingUploadFiles = logValue.GetPendingLogFilesToUploadForThisContainer
();
if (pendingUploadFiles.Count == 0)
{
return;
}
DataOutputStream @out = this.writer.PrepareAppendKey(-1);
logKey.Write(@out);
@out.Close();
@out = this.writer.PrepareAppendValue(-1);
logValue.Write(@out, pendingUploadFiles);
@out.Close();
}
public virtual AggregatedLogFormat.ContainerLogsReader GetContainerLogsReader(ContainerId
containerId)
{
AggregatedLogFormat.ContainerLogsReader logReader = null;
AggregatedLogFormat.LogKey containerKey = new AggregatedLogFormat.LogKey(containerId
);
AggregatedLogFormat.LogKey key = new AggregatedLogFormat.LogKey();
DataInputStream valueStream = Next(key);
while (valueStream != null && !key.Equals(containerKey))
{
valueStream = Next(key);
}
if (valueStream != null)
{
logReader = new AggregatedLogFormat.ContainerLogsReader(valueStream);
}
return(logReader);
}
/// <summary>Returns ACLs for the application.</summary>
/// <remarks>
/// Returns ACLs for the application. An empty map is returned if no ACLs are
/// found.
/// </remarks>
/// <returns>a map of the Application ACLs.</returns>
/// <exception cref="System.IO.IOException"/>
public virtual IDictionary <ApplicationAccessType, string> GetApplicationAcls()
{
// TODO Seek directly to the key once a comparator is specified.
TFile.Reader.Scanner aclScanner = reader.CreateScanner();
AggregatedLogFormat.LogKey key = new AggregatedLogFormat.LogKey();
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>();
while (!aclScanner.AtEnd())
{
TFile.Reader.Scanner.Entry entry = aclScanner.Entry();
key.ReadFields(entry.GetKeyStream());
if (key.ToString().Equals(ApplicationAclKey.ToString()))
{
DataInputStream valueStream = entry.GetValueStream();
while (true)
{
string appAccessOp = null;
string aclString = null;
try
{
appAccessOp = valueStream.ReadUTF();
}
catch (EOFException)
{
// Valid end of stream.
break;
}
try
{
aclString = valueStream.ReadUTF();
}
catch (EOFException e)
{
throw new YarnRuntimeException("Error reading ACLs", e);
}
acls[ApplicationAccessType.ValueOf(appAccessOp)] = aclString;
}
}
aclScanner.Advance();
}
return(acls);
}
public virtual int DumpAContainerLogs(string containerIdStr, AggregatedLogFormat.LogReader
reader, TextWriter @out, long logUploadedTime)
{
DataInputStream valueStream;
AggregatedLogFormat.LogKey key = new AggregatedLogFormat.LogKey();
valueStream = reader.Next(key);
while (valueStream != null && !key.ToString().Equals(containerIdStr))
{
// Next container
key = new AggregatedLogFormat.LogKey();
valueStream = reader.Next(key);
}
if (valueStream == null)
{
return(-1);
}
bool foundContainerLogs = false;
while (true)
{
try
{
AggregatedLogFormat.LogReader.ReadAContainerLogsForALogType(valueStream, @out, logUploadedTime
);
foundContainerLogs = true;
}
catch (EOFException)
{
break;
}
}
if (foundContainerLogs)
{
return(0);
}
return(-1);
}
/// <exception cref="System.Exception"/>
private void WriteSrcFileAndALog(Path srcFilePath, string fileName, long length,
Path remoteAppLogFile, Path srcFileRoot, ContainerId testContainerId)
{
FilePath dir = new FilePath(srcFilePath.ToString());
if (!dir.Exists())
{
if (!dir.Mkdirs())
{
throw new IOException("Unable to create directory : " + dir);
}
}
FilePath outputFile = new FilePath(new FilePath(srcFilePath.ToString()), fileName
);
FileOutputStream os = new FileOutputStream(outputFile);
OutputStreamWriter osw = new OutputStreamWriter(os, "UTF8");
int ch = filler;
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
AggregatedLogFormat.LogWriter logWriter = new AggregatedLogFormat.LogWriter(conf,
remoteAppLogFile, ugi);
AggregatedLogFormat.LogKey logKey = new AggregatedLogFormat.LogKey(testContainerId
);
AggregatedLogFormat.LogValue logValue = Org.Mockito.Mockito.Spy(new AggregatedLogFormat.LogValue
(Collections.SingletonList(srcFileRoot.ToString()), testContainerId, ugi.GetShortUserName
()));
CountDownLatch latch = new CountDownLatch(1);
Sharpen.Thread t = new _Thread_152(length, osw, ch, latch);
// TODO Auto-generated catch block
t.Start();
//Wait till the osw is partially written
//aggregation starts once the ows has completed 1/3rd of its work
latch.Await();
//Aggregate The Logs
logWriter.Append(logKey, logValue);
logWriter.Close();
}
public virtual int DumpAllContainersLogs(ApplicationId appId, string appOwner, TextWriter
@out)
{
Path remoteRootLogDir = new Path(GetConf().Get(YarnConfiguration.NmRemoteAppLogDir
, YarnConfiguration.DefaultNmRemoteAppLogDir));
string user = appOwner;
string logDirSuffix = LogAggregationUtils.GetRemoteNodeLogDirSuffix(GetConf());
// TODO Change this to get a list of files from the LAS.
Path remoteAppLogDir = LogAggregationUtils.GetRemoteAppLogDir(remoteRootLogDir, appId
, user, logDirSuffix);
RemoteIterator <FileStatus> nodeFiles;
try
{
Path qualifiedLogDir = FileContext.GetFileContext(GetConf()).MakeQualified(remoteAppLogDir
);
nodeFiles = FileContext.GetFileContext(qualifiedLogDir.ToUri(), GetConf()).ListStatus
(remoteAppLogDir);
}
catch (FileNotFoundException)
{
LogDirNotExist(remoteAppLogDir.ToString());
return(-1);
}
bool foundAnyLogs = false;
while (nodeFiles.HasNext())
{
FileStatus thisNodeFile = nodeFiles.Next();
if (!thisNodeFile.GetPath().GetName().EndsWith(LogAggregationUtils.TmpFileSuffix))
{
AggregatedLogFormat.LogReader reader = new AggregatedLogFormat.LogReader(GetConf(
), thisNodeFile.GetPath());
try
{
DataInputStream valueStream;
AggregatedLogFormat.LogKey key = new AggregatedLogFormat.LogKey();
valueStream = reader.Next(key);
while (valueStream != null)
{
string containerString = "\n\nContainer: " + key + " on " + thisNodeFile.GetPath(
).GetName();
@out.WriteLine(containerString);
@out.WriteLine(StringUtils.Repeat("=", containerString.Length));
while (true)
{
try
{
AggregatedLogFormat.LogReader.ReadAContainerLogsForALogType(valueStream, @out, thisNodeFile
.GetModificationTime());
foundAnyLogs = true;
}
catch (EOFException)
{
break;
}
}
// Next container
key = new AggregatedLogFormat.LogKey();
valueStream = reader.Next(key);
}
}
finally
{
reader.Close();
}
}
}
if (!foundAnyLogs)
{
EmptyLogDir(remoteAppLogDir.ToString());
return(-1);
}
return(0);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestContainerLogsFileAccess()
{
// This test will run only if NativeIO is enabled as SecureIOUtils
// require it to be enabled.
Assume.AssumeTrue(NativeIO.IsAvailable());
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
FilePath workDir = new FilePath(testWorkDir, "testContainerLogsFileAccess1");
Path remoteAppLogFile = new Path(workDir.GetAbsolutePath(), "aggregatedLogFile");
Path srcFileRoot = new Path(workDir.GetAbsolutePath(), "srcFiles");
string data = "Log File content for container : ";
// Creating files for container1. Log aggregator will try to read log files
// with illegal user.
ApplicationId applicationId = ApplicationId.NewInstance(1, 1);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 1);
ContainerId testContainerId1 = ContainerId.NewContainerId(applicationAttemptId, 1
);
Path appDir = new Path(srcFileRoot, testContainerId1.GetApplicationAttemptId().GetApplicationId
().ToString());
Path srcFilePath1 = new Path(appDir, testContainerId1.ToString());
string stdout = "stdout";
string stderr = "stderr";
WriteSrcFile(srcFilePath1, stdout, data + testContainerId1.ToString() + stdout);
WriteSrcFile(srcFilePath1, stderr, data + testContainerId1.ToString() + stderr);
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
AggregatedLogFormat.LogWriter logWriter = new AggregatedLogFormat.LogWriter(conf,
remoteAppLogFile, ugi);
AggregatedLogFormat.LogKey logKey = new AggregatedLogFormat.LogKey(testContainerId1
);
string randomUser = "******";
AggregatedLogFormat.LogValue logValue = Org.Mockito.Mockito.Spy(new AggregatedLogFormat.LogValue
(Collections.SingletonList(srcFileRoot.ToString()), testContainerId1, randomUser
));
// It is trying simulate a situation where first log file is owned by
// different user (probably symlink) and second one by the user itself.
// The first file should not be aggregated. Because this log file has the invalid
// user name.
Org.Mockito.Mockito.When(logValue.GetUser()).ThenReturn(randomUser).ThenReturn(ugi
.GetShortUserName());
logWriter.Append(logKey, logValue);
logWriter.Close();
BufferedReader @in = new BufferedReader(new FileReader(new FilePath(remoteAppLogFile
.ToUri().GetRawPath())));
string line;
StringBuilder sb = new StringBuilder(string.Empty);
while ((line = @in.ReadLine()) != null)
{
Log.Info(line);
sb.Append(line);
}
line = sb.ToString();
string expectedOwner = ugi.GetShortUserName();
if (Path.Windows)
{
string adminsGroupString = "Administrators";
if (Arrays.AsList(ugi.GetGroupNames()).Contains(adminsGroupString))
{
expectedOwner = adminsGroupString;
}
}
// This file: stderr should not be aggregated.
// And we will not aggregate the log message.
string stdoutFile1 = StringUtils.Join(FilePath.separator, Arrays.AsList(new string
[] { workDir.GetAbsolutePath(), "srcFiles", testContainerId1.GetApplicationAttemptId
().GetApplicationId().ToString(), testContainerId1.ToString(), stderr }));
// The file: stdout is expected to be aggregated.
string stdoutFile2 = StringUtils.Join(FilePath.separator, Arrays.AsList(new string
[] { workDir.GetAbsolutePath(), "srcFiles", testContainerId1.GetApplicationAttemptId
().GetApplicationId().ToString(), testContainerId1.ToString(), stdout }));
string message2 = "Owner '" + expectedOwner + "' for path " + stdoutFile2 + " did not match expected owner '"
+ ugi.GetShortUserName() + "'";
NUnit.Framework.Assert.IsFalse(line.Contains(message2));
NUnit.Framework.Assert.IsFalse(line.Contains(data + testContainerId1.ToString() +
stderr));
NUnit.Framework.Assert.IsTrue(line.Contains(data + testContainerId1.ToString() +
stdout));
}
/// <exception cref="System.Exception"/>
private void TestReadAcontainerLog(bool logUploadedTime)
{
Configuration conf = new Configuration();
FilePath workDir = new FilePath(testWorkDir, "testReadAcontainerLogs1");
Path remoteAppLogFile = new Path(workDir.GetAbsolutePath(), "aggregatedLogFile");
Path srcFileRoot = new Path(workDir.GetAbsolutePath(), "srcFiles");
ContainerId testContainerId = TestContainerId.NewContainerId(1, 1, 1, 1);
Path t = new Path(srcFileRoot, testContainerId.GetApplicationAttemptId().GetApplicationId
().ToString());
Path srcFilePath = new Path(t, testContainerId.ToString());
int numChars = 80000;
// create a sub-folder under srcFilePath
// and create file logs in this sub-folder.
// We only aggregate top level files.
// So, this log file should be ignored.
Path subDir = new Path(srcFilePath, "subDir");
fs.Mkdirs(subDir);
WriteSrcFile(subDir, "logs", numChars);
// create file stderr and stdout in containerLogDir
WriteSrcFile(srcFilePath, "stderr", numChars);
WriteSrcFile(srcFilePath, "stdout", numChars);
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
AggregatedLogFormat.LogWriter logWriter = new AggregatedLogFormat.LogWriter(conf,
remoteAppLogFile, ugi);
AggregatedLogFormat.LogKey logKey = new AggregatedLogFormat.LogKey(testContainerId
);
AggregatedLogFormat.LogValue logValue = new AggregatedLogFormat.LogValue(Collections
.SingletonList(srcFileRoot.ToString()), testContainerId, ugi.GetShortUserName());
// When we try to open FileInputStream for stderr, it will throw out an IOException.
// Skip the log aggregation for stderr.
AggregatedLogFormat.LogValue spyLogValue = Org.Mockito.Mockito.Spy(logValue);
FilePath errorFile = new FilePath((new Path(srcFilePath, "stderr")).ToString());
Org.Mockito.Mockito.DoThrow(new IOException("Mock can not open FileInputStream"))
.When(spyLogValue).SecureOpenFile(errorFile);
logWriter.Append(logKey, spyLogValue);
logWriter.Close();
// make sure permission are correct on the file
FileStatus fsStatus = fs.GetFileStatus(remoteAppLogFile);
NUnit.Framework.Assert.AreEqual("permissions on log aggregation file are wrong",
FsPermission.CreateImmutable((short)0x1a0), fsStatus.GetPermission());
AggregatedLogFormat.LogReader logReader = new AggregatedLogFormat.LogReader(conf,
remoteAppLogFile);
AggregatedLogFormat.LogKey rLogKey = new AggregatedLogFormat.LogKey();
DataInputStream dis = logReader.Next(rLogKey);
TextWriter writer = new StringWriter();
if (logUploadedTime)
{
AggregatedLogFormat.LogReader.ReadAcontainerLogs(dis, writer, Runtime.CurrentTimeMillis
());
}
else
{
AggregatedLogFormat.LogReader.ReadAcontainerLogs(dis, writer);
}
// We should only do the log aggregation for stdout.
// Since we could not open the fileInputStream for stderr, this file is not
// aggregated.
string s = writer.ToString();
int expectedLength = "LogType:stdout".Length + (logUploadedTime ? ("\nLog Upload Time:"
+ Times.Format(Runtime.CurrentTimeMillis())).Length : 0) + ("\nLogLength:" + numChars
).Length + "\nLog Contents:\n".Length + numChars + "\n".Length + "End of LogType:stdout\n"
.Length;
NUnit.Framework.Assert.IsTrue("LogType not matched", s.Contains("LogType:stdout")
);
NUnit.Framework.Assert.IsTrue("log file:stderr should not be aggregated.", !s.Contains
("LogType:stderr"));
NUnit.Framework.Assert.IsTrue("log file:logs should not be aggregated.", !s.Contains
("LogType:logs"));
NUnit.Framework.Assert.IsTrue("LogLength not matched", s.Contains("LogLength:" +
numChars));
NUnit.Framework.Assert.IsTrue("Log Contents not matched", s.Contains("Log Contents"
));
StringBuilder sb = new StringBuilder();
for (int i = 0; i < numChars; i++)
{
sb.Append(filler);
}
string expectedContent = sb.ToString();
NUnit.Framework.Assert.IsTrue("Log content incorrect", s.Contains(expectedContent
));
NUnit.Framework.Assert.AreEqual(expectedLength, s.Length);
}