public virtual void TestForCorruptedAggregatedLogs()
{
Configuration conf = new Configuration();
FilePath workDir = new FilePath(testWorkDir, "testReadAcontainerLogs1");
Path remoteAppLogFile = new Path(workDir.GetAbsolutePath(), "aggregatedLogFile");
Path srcFileRoot = new Path(workDir.GetAbsolutePath(), "srcFiles");
ContainerId testContainerId = TestContainerId.NewContainerId(1, 1, 1, 1);
Path t = new Path(srcFileRoot, testContainerId.GetApplicationAttemptId().GetApplicationId
().ToString());
Path srcFilePath = new Path(t, testContainerId.ToString());
long numChars = 950000;
WriteSrcFileAndALog(srcFilePath, "stdout", numChars, remoteAppLogFile, srcFileRoot
, testContainerId);
AggregatedLogFormat.LogReader logReader = new AggregatedLogFormat.LogReader(conf,
remoteAppLogFile);
AggregatedLogFormat.LogKey rLogKey = new AggregatedLogFormat.LogKey();
DataInputStream dis = logReader.Next(rLogKey);
TextWriter writer = new StringWriter();
try
{
AggregatedLogFormat.LogReader.ReadAcontainerLogs(dis, writer);
}
catch (Exception e)
{
if (e.ToString().Contains("NumberFormatException"))
{
NUnit.Framework.Assert.Fail("Aggregated logs are corrupted.");
}
}
}
/// <exception cref="System.Exception"/>
private void TestReadAcontainerLog(bool logUploadedTime)
{
Configuration conf = new Configuration();
FilePath workDir = new FilePath(testWorkDir, "testReadAcontainerLogs1");
Path remoteAppLogFile = new Path(workDir.GetAbsolutePath(), "aggregatedLogFile");
Path srcFileRoot = new Path(workDir.GetAbsolutePath(), "srcFiles");
ContainerId testContainerId = TestContainerId.NewContainerId(1, 1, 1, 1);
Path t = new Path(srcFileRoot, testContainerId.GetApplicationAttemptId().GetApplicationId
().ToString());
Path srcFilePath = new Path(t, testContainerId.ToString());
int numChars = 80000;
// create a sub-folder under srcFilePath
// and create file logs in this sub-folder.
// We only aggregate top level files.
// So, this log file should be ignored.
Path subDir = new Path(srcFilePath, "subDir");
fs.Mkdirs(subDir);
WriteSrcFile(subDir, "logs", numChars);
// create file stderr and stdout in containerLogDir
WriteSrcFile(srcFilePath, "stderr", numChars);
WriteSrcFile(srcFilePath, "stdout", numChars);
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
AggregatedLogFormat.LogWriter logWriter = new AggregatedLogFormat.LogWriter(conf,
remoteAppLogFile, ugi);
AggregatedLogFormat.LogKey logKey = new AggregatedLogFormat.LogKey(testContainerId
);
AggregatedLogFormat.LogValue logValue = new AggregatedLogFormat.LogValue(Collections
.SingletonList(srcFileRoot.ToString()), testContainerId, ugi.GetShortUserName());
// When we try to open FileInputStream for stderr, it will throw out an IOException.
// Skip the log aggregation for stderr.
AggregatedLogFormat.LogValue spyLogValue = Org.Mockito.Mockito.Spy(logValue);
FilePath errorFile = new FilePath((new Path(srcFilePath, "stderr")).ToString());
Org.Mockito.Mockito.DoThrow(new IOException("Mock can not open FileInputStream"))
.When(spyLogValue).SecureOpenFile(errorFile);
logWriter.Append(logKey, spyLogValue);
logWriter.Close();
// make sure permission are correct on the file
FileStatus fsStatus = fs.GetFileStatus(remoteAppLogFile);
NUnit.Framework.Assert.AreEqual("permissions on log aggregation file are wrong",
FsPermission.CreateImmutable((short)0x1a0), fsStatus.GetPermission());
AggregatedLogFormat.LogReader logReader = new AggregatedLogFormat.LogReader(conf,
remoteAppLogFile);
AggregatedLogFormat.LogKey rLogKey = new AggregatedLogFormat.LogKey();
DataInputStream dis = logReader.Next(rLogKey);
TextWriter writer = new StringWriter();
if (logUploadedTime)
{
AggregatedLogFormat.LogReader.ReadAcontainerLogs(dis, writer, Runtime.CurrentTimeMillis
());
}
else
{
AggregatedLogFormat.LogReader.ReadAcontainerLogs(dis, writer);
}
// We should only do the log aggregation for stdout.
// Since we could not open the fileInputStream for stderr, this file is not
// aggregated.
string s = writer.ToString();
int expectedLength = "LogType:stdout".Length + (logUploadedTime ? ("\nLog Upload Time:"
+ Times.Format(Runtime.CurrentTimeMillis())).Length : 0) + ("\nLogLength:" + numChars
).Length + "\nLog Contents:\n".Length + numChars + "\n".Length + "End of LogType:stdout\n"
.Length;
NUnit.Framework.Assert.IsTrue("LogType not matched", s.Contains("LogType:stdout")
);
NUnit.Framework.Assert.IsTrue("log file:stderr should not be aggregated.", !s.Contains
("LogType:stderr"));
NUnit.Framework.Assert.IsTrue("log file:logs should not be aggregated.", !s.Contains
("LogType:logs"));
NUnit.Framework.Assert.IsTrue("LogLength not matched", s.Contains("LogLength:" +
numChars));
NUnit.Framework.Assert.IsTrue("Log Contents not matched", s.Contains("Log Contents"
));
StringBuilder sb = new StringBuilder();
for (int i = 0; i < numChars; i++)
{
sb.Append(filler);
}
string expectedContent = sb.ToString();
NUnit.Framework.Assert.IsTrue("Log content incorrect", s.Contains(expectedContent
));
NUnit.Framework.Assert.AreEqual(expectedLength, s.Length);
}
