/// <exception cref="System.Exception"/>
private void WriteLog(Configuration configuration, string user)
{
ApplicationId appId = ApplicationIdPBImpl.NewInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.NewInstance(appId,
1);
ContainerId containerId = ContainerIdPBImpl.NewContainerId(appAttemptId, 1);
string path = "target/logs/" + user + "/logs/application_0_0001/localhost_1234";
FilePath f = new FilePath(path);
if (!f.GetParentFile().Exists())
{
NUnit.Framework.Assert.IsTrue(f.GetParentFile().Mkdirs());
}
IList <string> rootLogDirs = Arrays.AsList("target/logs/logs");
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
AggregatedLogFormat.LogWriter writer = new AggregatedLogFormat.LogWriter(configuration
, new Path(path), ugi);
writer.WriteApplicationOwner(ugi.GetUserName());
IDictionary <ApplicationAccessType, string> appAcls = new Dictionary <ApplicationAccessType
, string>();
appAcls[ApplicationAccessType.ViewApp] = ugi.GetUserName();
writer.WriteApplicationACLs(appAcls);
writer.Append(new AggregatedLogFormat.LogKey("container_0_0001_01_000001"), new AggregatedLogFormat.LogValue
(rootLogDirs, containerId, UserGroupInformation.GetCurrentUser().GetShortUserName
()));
writer.Close();
}
private Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job GetJob()
{
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
JobId jobId = new JobIdPBImpl();
ApplicationId appId = ApplicationIdPBImpl.NewInstance(Runtime.CurrentTimeMillis()
, 4);
jobId.SetAppId(appId);
jobId.SetId(1);
Org.Mockito.Mockito.When(job.GetID()).ThenReturn(jobId);
JobReport report = Org.Mockito.Mockito.Mock <JobReport>();
Org.Mockito.Mockito.When(report.GetStartTime()).ThenReturn(100010L);
Org.Mockito.Mockito.When(report.GetFinishTime()).ThenReturn(100015L);
Org.Mockito.Mockito.When(job.GetReport()).ThenReturn(report);
Org.Mockito.Mockito.When(job.GetName()).ThenReturn("JobName");
Org.Mockito.Mockito.When(job.GetUserName()).ThenReturn("UserName");
Org.Mockito.Mockito.When(job.GetQueueName()).ThenReturn("QueueName");
Org.Mockito.Mockito.When(job.GetState()).ThenReturn(JobState.Succeeded);
Org.Mockito.Mockito.When(job.GetTotalMaps()).ThenReturn(3);
Org.Mockito.Mockito.When(job.GetCompletedMaps()).ThenReturn(2);
Org.Mockito.Mockito.When(job.GetTotalReduces()).ThenReturn(2);
Org.Mockito.Mockito.When(job.GetCompletedReduces()).ThenReturn(1);
Org.Mockito.Mockito.When(job.GetCompletedReduces()).ThenReturn(1);
return(job);
}
public virtual void TestTasksBlock()
{
ApplicationId appId = ApplicationIdPBImpl.NewInstance(0, 1);
JobId jobId = new JobIdPBImpl();
jobId.SetId(0);
jobId.SetAppId(appId);
TaskId taskId = new TaskIdPBImpl();
taskId.SetId(0);
taskId.SetTaskType(TaskType.Map);
taskId.SetJobId(jobId);
Task task = Org.Mockito.Mockito.Mock <Task>();
Org.Mockito.Mockito.When(task.GetID()).ThenReturn(taskId);
TaskReport report = Org.Mockito.Mockito.Mock <TaskReport>();
Org.Mockito.Mockito.When(report.GetProgress()).ThenReturn(0.7f);
Org.Mockito.Mockito.When(report.GetTaskState()).ThenReturn(TaskState.Succeeded);
Org.Mockito.Mockito.When(report.GetStartTime()).ThenReturn(100001L);
Org.Mockito.Mockito.When(report.GetFinishTime()).ThenReturn(100011L);
Org.Mockito.Mockito.When(report.GetStatus()).ThenReturn("Dummy Status \n*");
Org.Mockito.Mockito.When(task.GetReport()).ThenReturn(report);
Org.Mockito.Mockito.When(task.GetType()).ThenReturn(TaskType.Map);
IDictionary <TaskId, Task> tasks = new Dictionary <TaskId, Task>();
tasks[taskId] = task;
AppContext ctx = Org.Mockito.Mockito.Mock <AppContext>();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
Org.Mockito.Mockito.When(job.GetTasks()).ThenReturn(tasks);
Org.Apache.Hadoop.Mapreduce.V2.App.Webapp.App app = new Org.Apache.Hadoop.Mapreduce.V2.App.Webapp.App
(ctx);
app.SetJob(job);
TasksBlockForTest taskBlock = new TasksBlockForTest(app);
taskBlock.AddParameter(AMParams.TaskType, "m");
PrintWriter pWriter = new PrintWriter(data);
HtmlBlock.Block html = new BlockForTest(new TestBlocks.HtmlBlockForTest(this), pWriter
, 0, false);
taskBlock.Render(html);
pWriter.Flush();
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("task_0_0001_m_000000"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("70.00"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("SUCCEEDED"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100001"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100011"));
NUnit.Framework.Assert.IsFalse(data.ToString().Contains("Dummy Status \n*"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("Dummy Status \\n*"));
}
private Task GetTask(long timestamp)
{
JobId jobId = new JobIdPBImpl();
jobId.SetId(0);
jobId.SetAppId(ApplicationIdPBImpl.NewInstance(timestamp, 1));
TaskId taskId = new TaskIdPBImpl();
taskId.SetId(0);
taskId.SetTaskType(TaskType.Reduce);
taskId.SetJobId(jobId);
Task task = Org.Mockito.Mockito.Mock <Task>();
Org.Mockito.Mockito.When(task.GetID()).ThenReturn(taskId);
TaskReport report = Org.Mockito.Mockito.Mock <TaskReport>();
Org.Mockito.Mockito.When(report.GetProgress()).ThenReturn(0.7f);
Org.Mockito.Mockito.When(report.GetTaskState()).ThenReturn(TaskState.Succeeded);
Org.Mockito.Mockito.When(report.GetStartTime()).ThenReturn(100001L);
Org.Mockito.Mockito.When(report.GetFinishTime()).ThenReturn(100011L);
Org.Mockito.Mockito.When(task.GetReport()).ThenReturn(report);
Org.Mockito.Mockito.When(task.GetType()).ThenReturn(TaskType.Reduce);
return(task);
}
/// <exception cref="System.Exception"/>
public virtual void TestFetchApplictionLogs()
{
string remoteLogRootDir = "target/logs/";
Configuration configuration = new Configuration();
configuration.SetBoolean(YarnConfiguration.LogAggregationEnabled, true);
configuration.Set(YarnConfiguration.NmRemoteAppLogDir, remoteLogRootDir);
configuration.SetBoolean(YarnConfiguration.YarnAclEnable, true);
configuration.Set(YarnConfiguration.YarnAdminAcl, "admin");
FileSystem fs = FileSystem.Get(configuration);
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
ApplicationId appId = ApplicationIdPBImpl.NewInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.NewInstance(appId,
1);
ContainerId containerId0 = ContainerIdPBImpl.NewContainerId(appAttemptId, 0);
ContainerId containerId1 = ContainerIdPBImpl.NewContainerId(appAttemptId, 1);
ContainerId containerId2 = ContainerIdPBImpl.NewContainerId(appAttemptId, 2);
NodeId nodeId = NodeId.NewInstance("localhost", 1234);
// create local logs
string rootLogDir = "target/LocalLogs";
Path rootLogDirPath = new Path(rootLogDir);
if (fs.Exists(rootLogDirPath))
{
fs.Delete(rootLogDirPath, true);
}
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(rootLogDirPath));
Path appLogsDir = new Path(rootLogDirPath, appId.ToString());
if (fs.Exists(appLogsDir))
{
fs.Delete(appLogsDir, true);
}
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(appLogsDir));
IList <string> rootLogDirs = Arrays.AsList(rootLogDir);
// create container logs in localLogDir
CreateContainerLogInLocalDir(appLogsDir, containerId1, fs);
CreateContainerLogInLocalDir(appLogsDir, containerId2, fs);
Path path = new Path(remoteLogRootDir + ugi.GetShortUserName() + "/logs/application_0_0001"
);
if (fs.Exists(path))
{
fs.Delete(path, true);
}
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path));
// upload container logs into remote directory
// the first two logs is empty. When we try to read first two logs,
// we will meet EOF exception, but it will not impact other logs.
// Other logs should be read successfully.
UploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId0
, path, fs);
UploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId1
, path, fs);
UploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId1
, path, fs);
UploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId2
, path, fs);
YarnClient mockYarnClient = CreateMockYarnClient(YarnApplicationState.Finished);
LogsCLI cli = new TestLogsCLI.LogsCLIForTest(mockYarnClient);
cli.SetConf(configuration);
int exitCode = cli.Run(new string[] { "-applicationId", appId.ToString() });
NUnit.Framework.Assert.IsTrue(exitCode == 0);
NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Hello container_0_0001_01_000001!"
));
NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Hello container_0_0001_01_000002!"
));
sysOutStream.Reset();
// uploaded two logs for container1. The first log is empty.
// The second one is not empty.
// We can still successfully read logs for container1.
exitCode = cli.Run(new string[] { "-applicationId", appId.ToString(), "-nodeAddress"
, nodeId.ToString(), "-containerId", containerId1.ToString() });
NUnit.Framework.Assert.IsTrue(exitCode == 0);
NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Hello container_0_0001_01_000001!"
));
NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Log Upload Time")
);
NUnit.Framework.Assert.IsTrue(!sysOutStream.ToString().Contains("Logs for container "
+ containerId1.ToString() + " are not present in this log-file."));
sysOutStream.Reset();
// Uploaded the empty log for container0.
// We should see the message showing the log for container0
// are not present.
exitCode = cli.Run(new string[] { "-applicationId", appId.ToString(), "-nodeAddress"
, nodeId.ToString(), "-containerId", containerId0.ToString() });
NUnit.Framework.Assert.IsTrue(exitCode == -1);
NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Logs for container "
+ containerId0.ToString() + " are not present in this log-file."));
fs.Delete(new Path(remoteLogRootDir), true);
fs.Delete(new Path(rootLogDir), true);
}
public virtual void TestAttemptsBlock()
{
AppContext ctx = Org.Mockito.Mockito.Mock <AppContext>();
AppForTest app = new AppForTest(ctx);
Task task = GetTask(0);
IDictionary <TaskAttemptId, TaskAttempt> attempts = new Dictionary <TaskAttemptId,
TaskAttempt>();
TaskAttempt attempt = Org.Mockito.Mockito.Mock <TaskAttempt>();
TaskAttemptId taId = new TaskAttemptIdPBImpl();
taId.SetId(0);
taId.SetTaskId(task.GetID());
Org.Mockito.Mockito.When(attempt.GetID()).ThenReturn(taId);
Org.Mockito.Mockito.When(attempt.GetNodeHttpAddress()).ThenReturn("Node address");
ApplicationId appId = ApplicationIdPBImpl.NewInstance(0, 5);
ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.NewInstance(appId,
1);
ContainerId containerId = ContainerIdPBImpl.NewContainerId(appAttemptId, 1);
Org.Mockito.Mockito.When(attempt.GetAssignedContainerID()).ThenReturn(containerId
);
Org.Mockito.Mockito.When(attempt.GetAssignedContainerMgrAddress()).ThenReturn("assignedContainerMgrAddress"
);
Org.Mockito.Mockito.When(attempt.GetNodeRackName()).ThenReturn("nodeRackName");
long taStartTime = 100002L;
long taFinishTime = 100012L;
long taShuffleFinishTime = 100010L;
long taSortFinishTime = 100011L;
TaskAttemptState taState = TaskAttemptState.Succeeded;
Org.Mockito.Mockito.When(attempt.GetLaunchTime()).ThenReturn(taStartTime);
Org.Mockito.Mockito.When(attempt.GetFinishTime()).ThenReturn(taFinishTime);
Org.Mockito.Mockito.When(attempt.GetShuffleFinishTime()).ThenReturn(taShuffleFinishTime
);
Org.Mockito.Mockito.When(attempt.GetSortFinishTime()).ThenReturn(taSortFinishTime
);
Org.Mockito.Mockito.When(attempt.GetState()).ThenReturn(taState);
TaskAttemptReport taReport = Org.Mockito.Mockito.Mock <TaskAttemptReport>();
Org.Mockito.Mockito.When(taReport.GetStartTime()).ThenReturn(taStartTime);
Org.Mockito.Mockito.When(taReport.GetFinishTime()).ThenReturn(taFinishTime);
Org.Mockito.Mockito.When(taReport.GetShuffleFinishTime()).ThenReturn(taShuffleFinishTime
);
Org.Mockito.Mockito.When(taReport.GetSortFinishTime()).ThenReturn(taSortFinishTime
);
Org.Mockito.Mockito.When(taReport.GetContainerId()).ThenReturn(containerId);
Org.Mockito.Mockito.When(taReport.GetProgress()).ThenReturn(1.0f);
Org.Mockito.Mockito.When(taReport.GetStateString()).ThenReturn("Processed 128/128 records <p> \n"
);
Org.Mockito.Mockito.When(taReport.GetTaskAttemptState()).ThenReturn(taState);
Org.Mockito.Mockito.When(taReport.GetDiagnosticInfo()).ThenReturn(string.Empty);
Org.Mockito.Mockito.When(attempt.GetReport()).ThenReturn(taReport);
attempts[taId] = attempt;
Org.Mockito.Mockito.When(task.GetAttempts()).ThenReturn(attempts);
app.SetTask(task);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
Org.Mockito.Mockito.When(job.GetUserName()).ThenReturn("User");
app.SetJob(job);
TestBlocks.AttemptsBlockForTest block = new TestBlocks.AttemptsBlockForTest(this,
app);
block.AddParameter(AMParams.TaskType, "r");
PrintWriter pWriter = new PrintWriter(data);
HtmlBlock.Block html = new BlockForTest(new TestBlocks.HtmlBlockForTest(this), pWriter
, 0, false);
block.Render(html);
pWriter.Flush();
// should be printed information about attempts
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("0 attempt_0_0001_r_000000_0"
));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("SUCCEEDED"));
NUnit.Framework.Assert.IsFalse(data.ToString().Contains("Processed 128/128 records <p> \n"
));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("Processed 128\\/128 records <p> \\n"
));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("_0005_01_000001:attempt_0_0001_r_000000_0:User:"******"100002"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100010"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100011"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100012"));
}
public virtual void TestHsController()
{
AppContext ctx = Org.Mockito.Mockito.Mock <AppContext>();
ApplicationId appId = ApplicationIdPBImpl.NewInstance(0, 5);
Org.Mockito.Mockito.When(ctx.GetApplicationID()).ThenReturn(appId);
AppForTest app = new AppForTest(ctx);
Configuration config = new Configuration();
Controller.RequestContext requestCtx = Org.Mockito.Mockito.Mock <Controller.RequestContext
>();
TestBlocks.HsControllerForTest controller = new TestBlocks.HsControllerForTest(app
, config, requestCtx);
controller.Index();
NUnit.Framework.Assert.AreEqual("JobHistory", controller.Get(Params.Title, string.Empty
));
NUnit.Framework.Assert.AreEqual(typeof(HsJobPage), controller.JobPage());
NUnit.Framework.Assert.AreEqual(typeof(HsCountersPage), controller.CountersPage()
);
NUnit.Framework.Assert.AreEqual(typeof(HsTasksPage), controller.TasksPage());
NUnit.Framework.Assert.AreEqual(typeof(HsTaskPage), controller.TaskPage());
NUnit.Framework.Assert.AreEqual(typeof(HsAttemptsPage), controller.AttemptsPage()
);
controller.Set(AMParams.JobId, "job_01_01");
controller.Set(AMParams.TaskId, "task_01_01_m01_01");
controller.Set(AMParams.TaskType, "m");
controller.Set(AMParams.AttemptState, "State");
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
Task task = Org.Mockito.Mockito.Mock <Task>();
Org.Mockito.Mockito.When(job.GetTask(Matchers.Any <TaskId>())).ThenReturn(task);
JobId jobID = MRApps.ToJobID("job_01_01");
Org.Mockito.Mockito.When(ctx.GetJob(jobID)).ThenReturn(job);
Org.Mockito.Mockito.When(job.CheckAccess(Matchers.Any <UserGroupInformation>(), Matchers.Any
<JobACL>())).ThenReturn(true);
controller.Job();
NUnit.Framework.Assert.AreEqual(typeof(HsJobPage), controller.GetClazz());
controller.JobCounters();
NUnit.Framework.Assert.AreEqual(typeof(HsCountersPage), controller.GetClazz());
controller.TaskCounters();
NUnit.Framework.Assert.AreEqual(typeof(HsCountersPage), controller.GetClazz());
controller.Tasks();
NUnit.Framework.Assert.AreEqual(typeof(HsTasksPage), controller.GetClazz());
controller.Task();
NUnit.Framework.Assert.AreEqual(typeof(HsTaskPage), controller.GetClazz());
controller.Attempts();
NUnit.Framework.Assert.AreEqual(typeof(HsAttemptsPage), controller.GetClazz());
NUnit.Framework.Assert.AreEqual(typeof(HsConfPage), controller.ConfPage());
NUnit.Framework.Assert.AreEqual(typeof(HsAboutPage), controller.AboutPage());
controller.About();
NUnit.Framework.Assert.AreEqual(typeof(HsAboutPage), controller.GetClazz());
controller.Logs();
NUnit.Framework.Assert.AreEqual(typeof(HsLogsPage), controller.GetClazz());
controller.Nmlogs();
NUnit.Framework.Assert.AreEqual(typeof(AggregatedLogsPage), controller.GetClazz()
);
NUnit.Framework.Assert.AreEqual(typeof(HsSingleCounterPage), controller.SingleCounterPage
());
controller.SingleJobCounter();
NUnit.Framework.Assert.AreEqual(typeof(HsSingleCounterPage), controller.GetClazz(
));
controller.SingleTaskCounter();
NUnit.Framework.Assert.AreEqual(typeof(HsSingleCounterPage), controller.GetClazz(
));
}