/// <summary>Read the next key and return the value-stream.</summary> /// <param name="key"/> /// <returns>the valueStream if there are more keys or null otherwise.</returns> /// <exception cref="System.IO.IOException"/> public virtual DataInputStream Next(AggregatedLogFormat.LogKey key) { if (!this.atBeginning) { this.scanner.Advance(); } else { this.atBeginning = false; } if (this.scanner.AtEnd()) { return(null); } TFile.Reader.Scanner.Entry entry = this.scanner.Entry(); key.ReadFields(entry.GetKeyStream()); // Skip META keys if (ReservedKeys.Contains(key.ToString())) { return(Next(key)); } DataInputStream valueStream = entry.GetValueStream(); return(valueStream); }
public virtual void TestForCorruptedAggregatedLogs() { Configuration conf = new Configuration(); FilePath workDir = new FilePath(testWorkDir, "testReadAcontainerLogs1"); Path remoteAppLogFile = new Path(workDir.GetAbsolutePath(), "aggregatedLogFile"); Path srcFileRoot = new Path(workDir.GetAbsolutePath(), "srcFiles"); ContainerId testContainerId = TestContainerId.NewContainerId(1, 1, 1, 1); Path t = new Path(srcFileRoot, testContainerId.GetApplicationAttemptId().GetApplicationId ().ToString()); Path srcFilePath = new Path(t, testContainerId.ToString()); long numChars = 950000; WriteSrcFileAndALog(srcFilePath, "stdout", numChars, remoteAppLogFile, srcFileRoot , testContainerId); AggregatedLogFormat.LogReader logReader = new AggregatedLogFormat.LogReader(conf, remoteAppLogFile); AggregatedLogFormat.LogKey rLogKey = new AggregatedLogFormat.LogKey(); DataInputStream dis = logReader.Next(rLogKey); TextWriter writer = new StringWriter(); try { AggregatedLogFormat.LogReader.ReadAcontainerLogs(dis, writer); } catch (Exception e) { if (e.ToString().Contains("NumberFormatException")) { NUnit.Framework.Assert.Fail("Aggregated logs are corrupted."); } } }
public override bool Equals(object obj) { if (obj is AggregatedLogFormat.LogKey) { AggregatedLogFormat.LogKey other = (AggregatedLogFormat.LogKey)obj; if (this.keyString == null) { return(other.keyString == null); } return(this.keyString.Equals(other.keyString)); } return(false); }
/// <summary>Returns the owner of the application.</summary> /// <returns>the application owner.</returns> /// <exception cref="System.IO.IOException"/> public virtual string GetApplicationOwner() { TFile.Reader.Scanner ownerScanner = reader.CreateScanner(); AggregatedLogFormat.LogKey key = new AggregatedLogFormat.LogKey(); while (!ownerScanner.AtEnd()) { TFile.Reader.Scanner.Entry entry = ownerScanner.Entry(); key.ReadFields(entry.GetKeyStream()); if (key.ToString().Equals(ApplicationOwnerKey.ToString())) { DataInputStream valueStream = entry.GetValueStream(); return(valueStream.ReadUTF()); } ownerScanner.Advance(); } return(null); }
/// <exception cref="System.IO.IOException"/> public virtual void Append(AggregatedLogFormat.LogKey logKey, AggregatedLogFormat.LogValue logValue) { ICollection <FilePath> pendingUploadFiles = logValue.GetPendingLogFilesToUploadForThisContainer (); if (pendingUploadFiles.Count == 0) { return; } DataOutputStream @out = this.writer.PrepareAppendKey(-1); logKey.Write(@out); @out.Close(); @out = this.writer.PrepareAppendValue(-1); logValue.Write(@out, pendingUploadFiles); @out.Close(); }
public virtual AggregatedLogFormat.ContainerLogsReader GetContainerLogsReader(ContainerId containerId) { AggregatedLogFormat.ContainerLogsReader logReader = null; AggregatedLogFormat.LogKey containerKey = new AggregatedLogFormat.LogKey(containerId ); AggregatedLogFormat.LogKey key = new AggregatedLogFormat.LogKey(); DataInputStream valueStream = Next(key); while (valueStream != null && !key.Equals(containerKey)) { valueStream = Next(key); } if (valueStream != null) { logReader = new AggregatedLogFormat.ContainerLogsReader(valueStream); } return(logReader); }
/// <summary>Returns ACLs for the application.</summary> /// <remarks> /// Returns ACLs for the application. An empty map is returned if no ACLs are /// found. /// </remarks> /// <returns>a map of the Application ACLs.</returns> /// <exception cref="System.IO.IOException"/> public virtual IDictionary <ApplicationAccessType, string> GetApplicationAcls() { // TODO Seek directly to the key once a comparator is specified. TFile.Reader.Scanner aclScanner = reader.CreateScanner(); AggregatedLogFormat.LogKey key = new AggregatedLogFormat.LogKey(); IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType , string>(); while (!aclScanner.AtEnd()) { TFile.Reader.Scanner.Entry entry = aclScanner.Entry(); key.ReadFields(entry.GetKeyStream()); if (key.ToString().Equals(ApplicationAclKey.ToString())) { DataInputStream valueStream = entry.GetValueStream(); while (true) { string appAccessOp = null; string aclString = null; try { appAccessOp = valueStream.ReadUTF(); } catch (EOFException) { // Valid end of stream. break; } try { aclString = valueStream.ReadUTF(); } catch (EOFException e) { throw new YarnRuntimeException("Error reading ACLs", e); } acls[ApplicationAccessType.ValueOf(appAccessOp)] = aclString; } } aclScanner.Advance(); } return(acls); }
public virtual int DumpAContainerLogs(string containerIdStr, AggregatedLogFormat.LogReader reader, TextWriter @out, long logUploadedTime) { DataInputStream valueStream; AggregatedLogFormat.LogKey key = new AggregatedLogFormat.LogKey(); valueStream = reader.Next(key); while (valueStream != null && !key.ToString().Equals(containerIdStr)) { // Next container key = new AggregatedLogFormat.LogKey(); valueStream = reader.Next(key); } if (valueStream == null) { return(-1); } bool foundContainerLogs = false; while (true) { try { AggregatedLogFormat.LogReader.ReadAContainerLogsForALogType(valueStream, @out, logUploadedTime ); foundContainerLogs = true; } catch (EOFException) { break; } } if (foundContainerLogs) { return(0); } return(-1); }
/// <exception cref="System.Exception"/> private void WriteSrcFileAndALog(Path srcFilePath, string fileName, long length, Path remoteAppLogFile, Path srcFileRoot, ContainerId testContainerId) { FilePath dir = new FilePath(srcFilePath.ToString()); if (!dir.Exists()) { if (!dir.Mkdirs()) { throw new IOException("Unable to create directory : " + dir); } } FilePath outputFile = new FilePath(new FilePath(srcFilePath.ToString()), fileName ); FileOutputStream os = new FileOutputStream(outputFile); OutputStreamWriter osw = new OutputStreamWriter(os, "UTF8"); int ch = filler; UserGroupInformation ugi = UserGroupInformation.GetCurrentUser(); AggregatedLogFormat.LogWriter logWriter = new AggregatedLogFormat.LogWriter(conf, remoteAppLogFile, ugi); AggregatedLogFormat.LogKey logKey = new AggregatedLogFormat.LogKey(testContainerId ); AggregatedLogFormat.LogValue logValue = Org.Mockito.Mockito.Spy(new AggregatedLogFormat.LogValue (Collections.SingletonList(srcFileRoot.ToString()), testContainerId, ugi.GetShortUserName ())); CountDownLatch latch = new CountDownLatch(1); Sharpen.Thread t = new _Thread_152(length, osw, ch, latch); // TODO Auto-generated catch block t.Start(); //Wait till the osw is partially written //aggregation starts once the ows has completed 1/3rd of its work latch.Await(); //Aggregate The Logs logWriter.Append(logKey, logValue); logWriter.Close(); }
public virtual int DumpAllContainersLogs(ApplicationId appId, string appOwner, TextWriter @out) { Path remoteRootLogDir = new Path(GetConf().Get(YarnConfiguration.NmRemoteAppLogDir , YarnConfiguration.DefaultNmRemoteAppLogDir)); string user = appOwner; string logDirSuffix = LogAggregationUtils.GetRemoteNodeLogDirSuffix(GetConf()); // TODO Change this to get a list of files from the LAS. Path remoteAppLogDir = LogAggregationUtils.GetRemoteAppLogDir(remoteRootLogDir, appId , user, logDirSuffix); RemoteIterator <FileStatus> nodeFiles; try { Path qualifiedLogDir = FileContext.GetFileContext(GetConf()).MakeQualified(remoteAppLogDir ); nodeFiles = FileContext.GetFileContext(qualifiedLogDir.ToUri(), GetConf()).ListStatus (remoteAppLogDir); } catch (FileNotFoundException) { LogDirNotExist(remoteAppLogDir.ToString()); return(-1); } bool foundAnyLogs = false; while (nodeFiles.HasNext()) { FileStatus thisNodeFile = nodeFiles.Next(); if (!thisNodeFile.GetPath().GetName().EndsWith(LogAggregationUtils.TmpFileSuffix)) { AggregatedLogFormat.LogReader reader = new AggregatedLogFormat.LogReader(GetConf( ), thisNodeFile.GetPath()); try { DataInputStream valueStream; AggregatedLogFormat.LogKey key = new AggregatedLogFormat.LogKey(); valueStream = reader.Next(key); while (valueStream != null) { string containerString = "\n\nContainer: " + key + " on " + thisNodeFile.GetPath( ).GetName(); @out.WriteLine(containerString); @out.WriteLine(StringUtils.Repeat("=", containerString.Length)); while (true) { try { AggregatedLogFormat.LogReader.ReadAContainerLogsForALogType(valueStream, @out, thisNodeFile .GetModificationTime()); foundAnyLogs = true; } catch (EOFException) { break; } } // Next container key = new AggregatedLogFormat.LogKey(); valueStream = reader.Next(key); } } finally { reader.Close(); } } } if (!foundAnyLogs) { EmptyLogDir(remoteAppLogDir.ToString()); return(-1); } return(0); }
/// <exception cref="System.IO.IOException"/> public virtual void TestContainerLogsFileAccess() { // This test will run only if NativeIO is enabled as SecureIOUtils // require it to be enabled. Assume.AssumeTrue(NativeIO.IsAvailable()); Configuration conf = new Configuration(); conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); FilePath workDir = new FilePath(testWorkDir, "testContainerLogsFileAccess1"); Path remoteAppLogFile = new Path(workDir.GetAbsolutePath(), "aggregatedLogFile"); Path srcFileRoot = new Path(workDir.GetAbsolutePath(), "srcFiles"); string data = "Log File content for container : "; // Creating files for container1. Log aggregator will try to read log files // with illegal user. ApplicationId applicationId = ApplicationId.NewInstance(1, 1); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId , 1); ContainerId testContainerId1 = ContainerId.NewContainerId(applicationAttemptId, 1 ); Path appDir = new Path(srcFileRoot, testContainerId1.GetApplicationAttemptId().GetApplicationId ().ToString()); Path srcFilePath1 = new Path(appDir, testContainerId1.ToString()); string stdout = "stdout"; string stderr = "stderr"; WriteSrcFile(srcFilePath1, stdout, data + testContainerId1.ToString() + stdout); WriteSrcFile(srcFilePath1, stderr, data + testContainerId1.ToString() + stderr); UserGroupInformation ugi = UserGroupInformation.GetCurrentUser(); AggregatedLogFormat.LogWriter logWriter = new AggregatedLogFormat.LogWriter(conf, remoteAppLogFile, ugi); AggregatedLogFormat.LogKey logKey = new AggregatedLogFormat.LogKey(testContainerId1 ); string randomUser = "******"; AggregatedLogFormat.LogValue logValue = Org.Mockito.Mockito.Spy(new AggregatedLogFormat.LogValue (Collections.SingletonList(srcFileRoot.ToString()), testContainerId1, randomUser )); // It is trying simulate a situation where first log file is owned by // different user (probably symlink) and second one by the user itself. // The first file should not be aggregated. Because this log file has the invalid // user name. Org.Mockito.Mockito.When(logValue.GetUser()).ThenReturn(randomUser).ThenReturn(ugi .GetShortUserName()); logWriter.Append(logKey, logValue); logWriter.Close(); BufferedReader @in = new BufferedReader(new FileReader(new FilePath(remoteAppLogFile .ToUri().GetRawPath()))); string line; StringBuilder sb = new StringBuilder(string.Empty); while ((line = @in.ReadLine()) != null) { Log.Info(line); sb.Append(line); } line = sb.ToString(); string expectedOwner = ugi.GetShortUserName(); if (Path.Windows) { string adminsGroupString = "Administrators"; if (Arrays.AsList(ugi.GetGroupNames()).Contains(adminsGroupString)) { expectedOwner = adminsGroupString; } } // This file: stderr should not be aggregated. // And we will not aggregate the log message. string stdoutFile1 = StringUtils.Join(FilePath.separator, Arrays.AsList(new string [] { workDir.GetAbsolutePath(), "srcFiles", testContainerId1.GetApplicationAttemptId ().GetApplicationId().ToString(), testContainerId1.ToString(), stderr })); // The file: stdout is expected to be aggregated. string stdoutFile2 = StringUtils.Join(FilePath.separator, Arrays.AsList(new string [] { workDir.GetAbsolutePath(), "srcFiles", testContainerId1.GetApplicationAttemptId ().GetApplicationId().ToString(), testContainerId1.ToString(), stdout })); string message2 = "Owner '" + expectedOwner + "' for path " + stdoutFile2 + " did not match expected owner '" + ugi.GetShortUserName() + "'"; NUnit.Framework.Assert.IsFalse(line.Contains(message2)); NUnit.Framework.Assert.IsFalse(line.Contains(data + testContainerId1.ToString() + stderr)); NUnit.Framework.Assert.IsTrue(line.Contains(data + testContainerId1.ToString() + stdout)); }
/// <exception cref="System.Exception"/> private void TestReadAcontainerLog(bool logUploadedTime) { Configuration conf = new Configuration(); FilePath workDir = new FilePath(testWorkDir, "testReadAcontainerLogs1"); Path remoteAppLogFile = new Path(workDir.GetAbsolutePath(), "aggregatedLogFile"); Path srcFileRoot = new Path(workDir.GetAbsolutePath(), "srcFiles"); ContainerId testContainerId = TestContainerId.NewContainerId(1, 1, 1, 1); Path t = new Path(srcFileRoot, testContainerId.GetApplicationAttemptId().GetApplicationId ().ToString()); Path srcFilePath = new Path(t, testContainerId.ToString()); int numChars = 80000; // create a sub-folder under srcFilePath // and create file logs in this sub-folder. // We only aggregate top level files. // So, this log file should be ignored. Path subDir = new Path(srcFilePath, "subDir"); fs.Mkdirs(subDir); WriteSrcFile(subDir, "logs", numChars); // create file stderr and stdout in containerLogDir WriteSrcFile(srcFilePath, "stderr", numChars); WriteSrcFile(srcFilePath, "stdout", numChars); UserGroupInformation ugi = UserGroupInformation.GetCurrentUser(); AggregatedLogFormat.LogWriter logWriter = new AggregatedLogFormat.LogWriter(conf, remoteAppLogFile, ugi); AggregatedLogFormat.LogKey logKey = new AggregatedLogFormat.LogKey(testContainerId ); AggregatedLogFormat.LogValue logValue = new AggregatedLogFormat.LogValue(Collections .SingletonList(srcFileRoot.ToString()), testContainerId, ugi.GetShortUserName()); // When we try to open FileInputStream for stderr, it will throw out an IOException. // Skip the log aggregation for stderr. AggregatedLogFormat.LogValue spyLogValue = Org.Mockito.Mockito.Spy(logValue); FilePath errorFile = new FilePath((new Path(srcFilePath, "stderr")).ToString()); Org.Mockito.Mockito.DoThrow(new IOException("Mock can not open FileInputStream")) .When(spyLogValue).SecureOpenFile(errorFile); logWriter.Append(logKey, spyLogValue); logWriter.Close(); // make sure permission are correct on the file FileStatus fsStatus = fs.GetFileStatus(remoteAppLogFile); NUnit.Framework.Assert.AreEqual("permissions on log aggregation file are wrong", FsPermission.CreateImmutable((short)0x1a0), fsStatus.GetPermission()); AggregatedLogFormat.LogReader logReader = new AggregatedLogFormat.LogReader(conf, remoteAppLogFile); AggregatedLogFormat.LogKey rLogKey = new AggregatedLogFormat.LogKey(); DataInputStream dis = logReader.Next(rLogKey); TextWriter writer = new StringWriter(); if (logUploadedTime) { AggregatedLogFormat.LogReader.ReadAcontainerLogs(dis, writer, Runtime.CurrentTimeMillis ()); } else { AggregatedLogFormat.LogReader.ReadAcontainerLogs(dis, writer); } // We should only do the log aggregation for stdout. // Since we could not open the fileInputStream for stderr, this file is not // aggregated. string s = writer.ToString(); int expectedLength = "LogType:stdout".Length + (logUploadedTime ? ("\nLog Upload Time:" + Times.Format(Runtime.CurrentTimeMillis())).Length : 0) + ("\nLogLength:" + numChars ).Length + "\nLog Contents:\n".Length + numChars + "\n".Length + "End of LogType:stdout\n" .Length; NUnit.Framework.Assert.IsTrue("LogType not matched", s.Contains("LogType:stdout") ); NUnit.Framework.Assert.IsTrue("log file:stderr should not be aggregated.", !s.Contains ("LogType:stderr")); NUnit.Framework.Assert.IsTrue("log file:logs should not be aggregated.", !s.Contains ("LogType:logs")); NUnit.Framework.Assert.IsTrue("LogLength not matched", s.Contains("LogLength:" + numChars)); NUnit.Framework.Assert.IsTrue("Log Contents not matched", s.Contains("Log Contents" )); StringBuilder sb = new StringBuilder(); for (int i = 0; i < numChars; i++) { sb.Append(filler); } string expectedContent = sb.ToString(); NUnit.Framework.Assert.IsTrue("Log content incorrect", s.Contains(expectedContent )); NUnit.Framework.Assert.AreEqual(expectedLength, s.Length); }