コード例 #1
0
        public virtual void TestForCorruptedAggregatedLogs()
        {
            Configuration conf             = new Configuration();
            FilePath      workDir          = new FilePath(testWorkDir, "testReadAcontainerLogs1");
            Path          remoteAppLogFile = new Path(workDir.GetAbsolutePath(), "aggregatedLogFile");
            Path          srcFileRoot      = new Path(workDir.GetAbsolutePath(), "srcFiles");
            ContainerId   testContainerId  = TestContainerId.NewContainerId(1, 1, 1, 1);
            Path          t = new Path(srcFileRoot, testContainerId.GetApplicationAttemptId().GetApplicationId
                                           ().ToString());
            Path srcFilePath = new Path(t, testContainerId.ToString());
            long numChars    = 950000;

            WriteSrcFileAndALog(srcFilePath, "stdout", numChars, remoteAppLogFile, srcFileRoot
                                , testContainerId);
            AggregatedLogFormat.LogReader logReader = new AggregatedLogFormat.LogReader(conf,
                                                                                        remoteAppLogFile);
            AggregatedLogFormat.LogKey rLogKey = new AggregatedLogFormat.LogKey();
            DataInputStream            dis     = logReader.Next(rLogKey);
            TextWriter writer = new StringWriter();

            try
            {
                AggregatedLogFormat.LogReader.ReadAcontainerLogs(dis, writer);
            }
            catch (Exception e)
            {
                if (e.ToString().Contains("NumberFormatException"))
                {
                    NUnit.Framework.Assert.Fail("Aggregated logs are corrupted.");
                }
            }
        }
コード例 #2
0
        /// <exception cref="System.Exception"/>
        private void TestReadAcontainerLog(bool logUploadedTime)
        {
            Configuration conf             = new Configuration();
            FilePath      workDir          = new FilePath(testWorkDir, "testReadAcontainerLogs1");
            Path          remoteAppLogFile = new Path(workDir.GetAbsolutePath(), "aggregatedLogFile");
            Path          srcFileRoot      = new Path(workDir.GetAbsolutePath(), "srcFiles");
            ContainerId   testContainerId  = TestContainerId.NewContainerId(1, 1, 1, 1);
            Path          t = new Path(srcFileRoot, testContainerId.GetApplicationAttemptId().GetApplicationId
                                           ().ToString());
            Path srcFilePath = new Path(t, testContainerId.ToString());
            int  numChars    = 80000;
            // create a sub-folder under srcFilePath
            // and create file logs in this sub-folder.
            // We only aggregate top level files.
            // So, this log file should be ignored.
            Path subDir = new Path(srcFilePath, "subDir");

            fs.Mkdirs(subDir);
            WriteSrcFile(subDir, "logs", numChars);
            // create file stderr and stdout in containerLogDir
            WriteSrcFile(srcFilePath, "stderr", numChars);
            WriteSrcFile(srcFilePath, "stdout", numChars);
            UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();

            AggregatedLogFormat.LogWriter logWriter = new AggregatedLogFormat.LogWriter(conf,
                                                                                        remoteAppLogFile, ugi);
            AggregatedLogFormat.LogKey logKey = new AggregatedLogFormat.LogKey(testContainerId
                                                                               );
            AggregatedLogFormat.LogValue logValue = new AggregatedLogFormat.LogValue(Collections
                                                                                     .SingletonList(srcFileRoot.ToString()), testContainerId, ugi.GetShortUserName());
            // When we try to open FileInputStream for stderr, it will throw out an IOException.
            // Skip the log aggregation for stderr.
            AggregatedLogFormat.LogValue spyLogValue = Org.Mockito.Mockito.Spy(logValue);
            FilePath errorFile = new FilePath((new Path(srcFilePath, "stderr")).ToString());

            Org.Mockito.Mockito.DoThrow(new IOException("Mock can not open FileInputStream"))
            .When(spyLogValue).SecureOpenFile(errorFile);
            logWriter.Append(logKey, spyLogValue);
            logWriter.Close();
            // make sure permission are correct on the file
            FileStatus fsStatus = fs.GetFileStatus(remoteAppLogFile);

            NUnit.Framework.Assert.AreEqual("permissions on log aggregation file are wrong",
                                            FsPermission.CreateImmutable((short)0x1a0), fsStatus.GetPermission());
            AggregatedLogFormat.LogReader logReader = new AggregatedLogFormat.LogReader(conf,
                                                                                        remoteAppLogFile);
            AggregatedLogFormat.LogKey rLogKey = new AggregatedLogFormat.LogKey();
            DataInputStream            dis     = logReader.Next(rLogKey);
            TextWriter writer = new StringWriter();

            if (logUploadedTime)
            {
                AggregatedLogFormat.LogReader.ReadAcontainerLogs(dis, writer, Runtime.CurrentTimeMillis
                                                                     ());
            }
            else
            {
                AggregatedLogFormat.LogReader.ReadAcontainerLogs(dis, writer);
            }
            // We should only do the log aggregation for stdout.
            // Since we could not open the fileInputStream for stderr, this file is not
            // aggregated.
            string s = writer.ToString();
            int    expectedLength = "LogType:stdout".Length + (logUploadedTime ? ("\nLog Upload Time:"
                                                                                  + Times.Format(Runtime.CurrentTimeMillis())).Length : 0) + ("\nLogLength:" + numChars
                                                                                                                                              ).Length + "\nLog Contents:\n".Length + numChars + "\n".Length + "End of LogType:stdout\n"
                                    .Length;

            NUnit.Framework.Assert.IsTrue("LogType not matched", s.Contains("LogType:stdout")
                                          );
            NUnit.Framework.Assert.IsTrue("log file:stderr should not be aggregated.", !s.Contains
                                              ("LogType:stderr"));
            NUnit.Framework.Assert.IsTrue("log file:logs should not be aggregated.", !s.Contains
                                              ("LogType:logs"));
            NUnit.Framework.Assert.IsTrue("LogLength not matched", s.Contains("LogLength:" +
                                                                              numChars));
            NUnit.Framework.Assert.IsTrue("Log Contents not matched", s.Contains("Log Contents"
                                                                                 ));
            StringBuilder sb = new StringBuilder();

            for (int i = 0; i < numChars; i++)
            {
                sb.Append(filler);
            }
            string expectedContent = sb.ToString();

            NUnit.Framework.Assert.IsTrue("Log content incorrect", s.Contains(expectedContent
                                                                              ));
            NUnit.Framework.Assert.AreEqual(expectedLength, s.Length);
        }