/// <exception cref="System.Exception"/>
private void WriteLog(Configuration configuration, string user)
{
ApplicationId appId = ApplicationIdPBImpl.NewInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.NewInstance(appId,
1);
ContainerId containerId = ContainerIdPBImpl.NewContainerId(appAttemptId, 1);
string path = "target/logs/" + user + "/logs/application_0_0001/localhost_1234";
FilePath f = new FilePath(path);
if (!f.GetParentFile().Exists())
{
NUnit.Framework.Assert.IsTrue(f.GetParentFile().Mkdirs());
}
IList <string> rootLogDirs = Arrays.AsList("target/logs/logs");
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
AggregatedLogFormat.LogWriter writer = new AggregatedLogFormat.LogWriter(configuration
, new Path(path), ugi);
writer.WriteApplicationOwner(ugi.GetUserName());
IDictionary <ApplicationAccessType, string> appAcls = new Dictionary <ApplicationAccessType
, string>();
appAcls[ApplicationAccessType.ViewApp] = ugi.GetUserName();
writer.WriteApplicationACLs(appAcls);
writer.Append(new AggregatedLogFormat.LogKey("container_0_0001_01_000001"), new AggregatedLogFormat.LogValue
(rootLogDirs, containerId, UserGroupInformation.GetCurrentUser().GetShortUserName
()));
writer.Close();
}
private ContainerId GetContainerId(int containerID, int appAttemptId)
{
ContainerId containerId = ContainerIdPBImpl.NewContainerId(GetApplicationAttemptId
(appAttemptId), containerID);
return(containerId);
}
public override UserGroupInformation GetUser()
{
string containerId = null;
if (proto.HasContainerId())
{
containerId = new ContainerIdPBImpl(proto.GetContainerId()).ToString();
}
return(UserGroupInformation.CreateRemoteUser(containerId));
}
public ContainerTokenIdentifierForTest(ContainerTokenIdentifier identifier, string
message)
{
YarnSecurityTestTokenProtos.ContainerTokenIdentifierForTestProto.Builder builder =
YarnSecurityTestTokenProtos.ContainerTokenIdentifierForTestProto.NewBuilder();
ContainerIdPBImpl containerID = (ContainerIdPBImpl)identifier.GetContainerID();
if (containerID != null)
{
builder.SetContainerId(containerID.GetProto());
}
builder.SetNmHostAddr(identifier.GetNmHostAddress());
builder.SetAppSubmitter(identifier.GetApplicationSubmitter());
ResourcePBImpl resource = (ResourcePBImpl)identifier.GetResource();
if (resource != null)
{
builder.SetResource(resource.GetProto());
}
builder.SetExpiryTimeStamp(identifier.GetExpiryTimeStamp());
builder.SetMasterKeyId(identifier.GetMasterKeyId());
builder.SetRmIdentifier(identifier.GetRMIdentifier());
PriorityPBImpl priority = (PriorityPBImpl)identifier.GetPriority();
if (priority != null)
{
builder.SetPriority(priority.GetProto());
}
builder.SetCreationTime(identifier.GetCreationTime());
builder.SetMessage(message);
LogAggregationContextPBImpl logAggregationContext = (LogAggregationContextPBImpl)
identifier.GetLogAggregationContext();
if (logAggregationContext != null)
{
builder.SetLogAggregationContext(logAggregationContext.GetProto());
}
proto = ((YarnSecurityTestTokenProtos.ContainerTokenIdentifierForTestProto)builder
.Build());
}
/// <exception cref="System.Exception"/>
public virtual void TestFetchApplictionLogs()
{
string remoteLogRootDir = "target/logs/";
Configuration configuration = new Configuration();
configuration.SetBoolean(YarnConfiguration.LogAggregationEnabled, true);
configuration.Set(YarnConfiguration.NmRemoteAppLogDir, remoteLogRootDir);
configuration.SetBoolean(YarnConfiguration.YarnAclEnable, true);
configuration.Set(YarnConfiguration.YarnAdminAcl, "admin");
FileSystem fs = FileSystem.Get(configuration);
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
ApplicationId appId = ApplicationIdPBImpl.NewInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.NewInstance(appId,
1);
ContainerId containerId0 = ContainerIdPBImpl.NewContainerId(appAttemptId, 0);
ContainerId containerId1 = ContainerIdPBImpl.NewContainerId(appAttemptId, 1);
ContainerId containerId2 = ContainerIdPBImpl.NewContainerId(appAttemptId, 2);
NodeId nodeId = NodeId.NewInstance("localhost", 1234);
// create local logs
string rootLogDir = "target/LocalLogs";
Path rootLogDirPath = new Path(rootLogDir);
if (fs.Exists(rootLogDirPath))
{
fs.Delete(rootLogDirPath, true);
}
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(rootLogDirPath));
Path appLogsDir = new Path(rootLogDirPath, appId.ToString());
if (fs.Exists(appLogsDir))
{
fs.Delete(appLogsDir, true);
}
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(appLogsDir));
IList <string> rootLogDirs = Arrays.AsList(rootLogDir);
// create container logs in localLogDir
CreateContainerLogInLocalDir(appLogsDir, containerId1, fs);
CreateContainerLogInLocalDir(appLogsDir, containerId2, fs);
Path path = new Path(remoteLogRootDir + ugi.GetShortUserName() + "/logs/application_0_0001"
);
if (fs.Exists(path))
{
fs.Delete(path, true);
}
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path));
// upload container logs into remote directory
// the first two logs is empty. When we try to read first two logs,
// we will meet EOF exception, but it will not impact other logs.
// Other logs should be read successfully.
UploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId0
, path, fs);
UploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId1
, path, fs);
UploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId1
, path, fs);
UploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId2
, path, fs);
YarnClient mockYarnClient = CreateMockYarnClient(YarnApplicationState.Finished);
LogsCLI cli = new TestLogsCLI.LogsCLIForTest(mockYarnClient);
cli.SetConf(configuration);
int exitCode = cli.Run(new string[] { "-applicationId", appId.ToString() });
NUnit.Framework.Assert.IsTrue(exitCode == 0);
NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Hello container_0_0001_01_000001!"
));
NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Hello container_0_0001_01_000002!"
));
sysOutStream.Reset();
// uploaded two logs for container1. The first log is empty.
// The second one is not empty.
// We can still successfully read logs for container1.
exitCode = cli.Run(new string[] { "-applicationId", appId.ToString(), "-nodeAddress"
, nodeId.ToString(), "-containerId", containerId1.ToString() });
NUnit.Framework.Assert.IsTrue(exitCode == 0);
NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Hello container_0_0001_01_000001!"
));
NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Log Upload Time")
);
NUnit.Framework.Assert.IsTrue(!sysOutStream.ToString().Contains("Logs for container "
+ containerId1.ToString() + " are not present in this log-file."));
sysOutStream.Reset();
// Uploaded the empty log for container0.
// We should see the message showing the log for container0
// are not present.
exitCode = cli.Run(new string[] { "-applicationId", appId.ToString(), "-nodeAddress"
, nodeId.ToString(), "-containerId", containerId0.ToString() });
NUnit.Framework.Assert.IsTrue(exitCode == -1);
NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Logs for container "
+ containerId0.ToString() + " are not present in this log-file."));
fs.Delete(new Path(remoteLogRootDir), true);
fs.Delete(new Path(rootLogDir), true);
}
public virtual void TestAttemptsBlock()
{
AppContext ctx = Org.Mockito.Mockito.Mock <AppContext>();
AppForTest app = new AppForTest(ctx);
Task task = GetTask(0);
IDictionary <TaskAttemptId, TaskAttempt> attempts = new Dictionary <TaskAttemptId,
TaskAttempt>();
TaskAttempt attempt = Org.Mockito.Mockito.Mock <TaskAttempt>();
TaskAttemptId taId = new TaskAttemptIdPBImpl();
taId.SetId(0);
taId.SetTaskId(task.GetID());
Org.Mockito.Mockito.When(attempt.GetID()).ThenReturn(taId);
Org.Mockito.Mockito.When(attempt.GetNodeHttpAddress()).ThenReturn("Node address");
ApplicationId appId = ApplicationIdPBImpl.NewInstance(0, 5);
ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.NewInstance(appId,
1);
ContainerId containerId = ContainerIdPBImpl.NewContainerId(appAttemptId, 1);
Org.Mockito.Mockito.When(attempt.GetAssignedContainerID()).ThenReturn(containerId
);
Org.Mockito.Mockito.When(attempt.GetAssignedContainerMgrAddress()).ThenReturn("assignedContainerMgrAddress"
);
Org.Mockito.Mockito.When(attempt.GetNodeRackName()).ThenReturn("nodeRackName");
long taStartTime = 100002L;
long taFinishTime = 100012L;
long taShuffleFinishTime = 100010L;
long taSortFinishTime = 100011L;
TaskAttemptState taState = TaskAttemptState.Succeeded;
Org.Mockito.Mockito.When(attempt.GetLaunchTime()).ThenReturn(taStartTime);
Org.Mockito.Mockito.When(attempt.GetFinishTime()).ThenReturn(taFinishTime);
Org.Mockito.Mockito.When(attempt.GetShuffleFinishTime()).ThenReturn(taShuffleFinishTime
);
Org.Mockito.Mockito.When(attempt.GetSortFinishTime()).ThenReturn(taSortFinishTime
);
Org.Mockito.Mockito.When(attempt.GetState()).ThenReturn(taState);
TaskAttemptReport taReport = Org.Mockito.Mockito.Mock <TaskAttemptReport>();
Org.Mockito.Mockito.When(taReport.GetStartTime()).ThenReturn(taStartTime);
Org.Mockito.Mockito.When(taReport.GetFinishTime()).ThenReturn(taFinishTime);
Org.Mockito.Mockito.When(taReport.GetShuffleFinishTime()).ThenReturn(taShuffleFinishTime
);
Org.Mockito.Mockito.When(taReport.GetSortFinishTime()).ThenReturn(taSortFinishTime
);
Org.Mockito.Mockito.When(taReport.GetContainerId()).ThenReturn(containerId);
Org.Mockito.Mockito.When(taReport.GetProgress()).ThenReturn(1.0f);
Org.Mockito.Mockito.When(taReport.GetStateString()).ThenReturn("Processed 128/128 records <p> \n"
);
Org.Mockito.Mockito.When(taReport.GetTaskAttemptState()).ThenReturn(taState);
Org.Mockito.Mockito.When(taReport.GetDiagnosticInfo()).ThenReturn(string.Empty);
Org.Mockito.Mockito.When(attempt.GetReport()).ThenReturn(taReport);
attempts[taId] = attempt;
Org.Mockito.Mockito.When(task.GetAttempts()).ThenReturn(attempts);
app.SetTask(task);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
Org.Mockito.Mockito.When(job.GetUserName()).ThenReturn("User");
app.SetJob(job);
TestBlocks.AttemptsBlockForTest block = new TestBlocks.AttemptsBlockForTest(this,
app);
block.AddParameter(AMParams.TaskType, "r");
PrintWriter pWriter = new PrintWriter(data);
HtmlBlock.Block html = new BlockForTest(new TestBlocks.HtmlBlockForTest(this), pWriter
, 0, false);
block.Render(html);
pWriter.Flush();
// should be printed information about attempts
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("0 attempt_0_0001_r_000000_0"
));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("SUCCEEDED"));
NUnit.Framework.Assert.IsFalse(data.ToString().Contains("Processed 128/128 records <p> \n"
));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("Processed 128\\/128 records <p> \\n"
));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("_0005_01_000001:attempt_0_0001_r_000000_0:User:"******"100002"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100010"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100011"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100012"));
}
