public virtual void TestLogsViewSingle()
{
Log.Info("HsLogsPage with params for single log and data limits");
MockAppContext ctx = new MockAppContext(0, 1, 1, 1);
IDictionary <string, string> @params = new Dictionary <string, string>();
Configuration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.LogAggregationEnabled, true);
@params["start"] = "-2048";
@params["end"] = "-1024";
@params[YarnWebParams.ContainerLogType] = "syslog";
@params[YarnWebParams.ContainerId] = MRApp.NewContainerId(1, 1, 333, 1).ToString(
);
@params[YarnWebParams.NmNodename] = NodeId.NewInstance(MockJobs.NmHost, MockJobs.
NmPort).ToString();
@params[YarnWebParams.EntityString] = "container_10_0001_01_000001";
@params[YarnWebParams.AppOwner] = "owner";
Injector injector = WebAppTests.TestPage <AppContext>(typeof(AggregatedLogsPage),
ctx, @params, new _AbstractModule_201(conf));
PrintWriter spyPw = WebAppTests.GetPrintWriter(injector);
Org.Mockito.Mockito.Verify(spyPw).Write("Logs not available for container_10_0001_01_000001."
+ " Aggregation may not be complete, " + "Check back later or try the nodemanager at "
+ MockJobs.NmHost + ":" + MockJobs.NmPort);
}
/// <exception cref="System.IO.IOException"/>
public virtual StartContainersResponse StartContainers(StartContainersRequest requests
)
{
StartContainerRequest request = requests.GetStartContainerRequests()[0];
ContainerTokenIdentifier containerTokenIdentifier = MRApp.NewContainerTokenIdentifier
(request.GetContainerToken());
// Validate that the container is what RM is giving.
NUnit.Framework.Assert.AreEqual(MRApp.NmHost + ":" + MRApp.NmPort, containerTokenIdentifier
.GetNmHostAddress());
StartContainersResponse response = TestContainerLauncher.recordFactory.NewRecordInstance
<StartContainersResponse>();
this.status = TestContainerLauncher.recordFactory.NewRecordInstance<ContainerStatus
>();
try
{
// make the thread sleep to look like its not going to respond
Sharpen.Thread.Sleep(15000);
}
catch (Exception e)
{
TestContainerLauncher.Log.Error(e);
throw new UndeclaredThrowableException(e);
}
this.status.SetState(ContainerState.Running);
this.status.SetContainerId(containerTokenIdentifier.GetContainerID());
this.status.SetExitStatus(0);
return response;
}
private Token CreateNewContainerToken(ContainerId contId, string containerManagerAddr
)
{
long currentTime = Runtime.CurrentTimeMillis();
return(MRApp.NewContainerToken(NodeId.NewInstance("127.0.0.1", 1234), Sharpen.Runtime.GetBytesForString
("password"), new ContainerTokenIdentifier(contId, containerManagerAddr, "user",
Resource.NewInstance(1024, 1), currentTime + 10000L, 123, currentTime, Priority.
NewInstance(0), 0)));
}
private void VerifyAttempt(TaskAttempt attempt)
{
NUnit.Framework.Assert.AreEqual("TaskAttempt state not currect", TaskAttemptState
.Succeeded, attempt.GetState());
NUnit.Framework.Assert.IsNotNull(attempt.GetAssignedContainerID());
//Verify the wrong ctor is not being used. Remove after mrv1 is removed.
ContainerId fakeCid = MRApp.NewContainerId(-1, -1, -1, -1);
NUnit.Framework.Assert.IsFalse(attempt.GetAssignedContainerID().Equals(fakeCid));
//Verify complete contianerManagerAddress
NUnit.Framework.Assert.AreEqual(MRApp.NmHost + ":" + MRApp.NmPort, attempt.GetAssignedContainerMgrAddress
());
}
public virtual void TestLogsView2()
{
Log.Info("HsLogsPage with data");
MockAppContext ctx = new MockAppContext(0, 1, 1, 1);
IDictionary <string, string> @params = new Dictionary <string, string>();
@params[YarnWebParams.ContainerId] = MRApp.NewContainerId(1, 1, 333, 1).ToString(
);
@params[YarnWebParams.NmNodename] = NodeId.NewInstance(MockJobs.NmHost, MockJobs.
NmPort).ToString();
@params[YarnWebParams.EntityString] = "container_10_0001_01_000001";
@params[YarnWebParams.AppOwner] = "owner";
Injector injector = WebAppTests.TestPage <AppContext>(typeof(AggregatedLogsPage),
ctx, @params);
PrintWriter spyPw = WebAppTests.GetPrintWriter(injector);
Org.Mockito.Mockito.Verify(spyPw).Write("Aggregation is not enabled. Try the nodemanager at "
+ MockJobs.NmHost + ":" + MockJobs.NmPort);
}
public virtual void TestLogsViewBadStartEnd()
{
Log.Info("HsLogsPage with bad start/end params");
MockAppContext ctx = new MockAppContext(0, 1, 1, 1);
IDictionary <string, string> @params = new Dictionary <string, string>();
@params["start"] = "foo";
@params["end"] = "bar";
@params[YarnWebParams.ContainerId] = MRApp.NewContainerId(1, 1, 333, 1).ToString(
);
@params[YarnWebParams.NmNodename] = NodeId.NewInstance(MockJobs.NmHost, MockJobs.
NmPort).ToString();
@params[YarnWebParams.EntityString] = "container_10_0001_01_000001";
@params[YarnWebParams.AppOwner] = "owner";
Injector injector = WebAppTests.TestPage <AppContext>(typeof(AggregatedLogsPage),
ctx, @params);
PrintWriter spyPw = WebAppTests.GetPrintWriter(injector);
Org.Mockito.Mockito.Verify(spyPw).Write("Invalid log start value: foo");
Org.Mockito.Mockito.Verify(spyPw).Write("Invalid log end value: bar");
}
/// <exception cref="System.Exception"/>
private static void VerifySpeculationMessage(MRApp app, TaskAttempt[] ta)
{
app.WaitForState(ta[0], TaskAttemptState.Succeeded);
}
public virtual void TestSpeculateSuccessfulWithoutUpdateEvents()
{
Clock actualClock = new SystemClock();
ControlledClock clock = new ControlledClock(actualClock);
clock.SetTime(Runtime.CurrentTimeMillis());
MRApp app = new MRApp(NumMappers, NumReducers, false, "test", true, clock);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration(),
true, true);
app.WaitForState(job, JobState.Running);
IDictionary <TaskId, Task> tasks = job.GetTasks();
NUnit.Framework.Assert.AreEqual("Num tasks is not correct", NumMappers + NumReducers
, tasks.Count);
IEnumerator <Task> taskIter = tasks.Values.GetEnumerator();
while (taskIter.HasNext())
{
app.WaitForState(taskIter.Next(), TaskState.Running);
}
// Process the update events
clock.SetTime(Runtime.CurrentTimeMillis() + 2000);
EventHandler appEventHandler = app.GetContext().GetEventHandler();
foreach (KeyValuePair <TaskId, Task> mapTask in tasks)
{
foreach (KeyValuePair <TaskAttemptId, TaskAttempt> taskAttempt in mapTask.Value.GetAttempts
())
{
TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = CreateTaskAttemptStatus(taskAttempt
.Key, (float)0.8, TaskAttemptState.Running);
TaskAttemptStatusUpdateEvent @event = new TaskAttemptStatusUpdateEvent(taskAttempt
.Key, status);
appEventHandler.Handle(@event);
}
}
Random generator = new Random();
object[] taskValues = Sharpen.Collections.ToArray(tasks.Values);
Task taskToBeSpeculated = (Task)taskValues[generator.Next(taskValues.Length)];
// Other than one random task, finish every other task.
foreach (KeyValuePair <TaskId, Task> mapTask_1 in tasks)
{
foreach (KeyValuePair <TaskAttemptId, TaskAttempt> taskAttempt in mapTask_1.Value.
GetAttempts())
{
if (mapTask_1.Key != taskToBeSpeculated.GetID())
{
appEventHandler.Handle(new TaskAttemptEvent(taskAttempt.Key, TaskAttemptEventType
.TaDone));
appEventHandler.Handle(new TaskAttemptEvent(taskAttempt.Key, TaskAttemptEventType
.TaContainerCleaned));
app.WaitForState(taskAttempt.Value, TaskAttemptState.Succeeded);
}
}
}
GenericTestUtils.WaitFor(new _Supplier_111(taskToBeSpeculated, clock), 1000, 60000
);
// finish 1st TA, 2nd will be killed
TaskAttempt[] ta = MakeFirstAttemptWin(appEventHandler, taskToBeSpeculated);
VerifySpeculationMessage(app, ta);
app.WaitForState(Service.STATE.Stopped);
}
public virtual void TestSepculateSuccessfulWithUpdateEvents()
{
Clock actualClock = new SystemClock();
ControlledClock clock = new ControlledClock(actualClock);
clock.SetTime(Runtime.CurrentTimeMillis());
MRApp app = new MRApp(NumMappers, NumReducers, false, "test", true, clock);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration(),
true, true);
app.WaitForState(job, JobState.Running);
IDictionary <TaskId, Task> tasks = job.GetTasks();
NUnit.Framework.Assert.AreEqual("Num tasks is not correct", NumMappers + NumReducers
, tasks.Count);
IEnumerator <Task> taskIter = tasks.Values.GetEnumerator();
while (taskIter.HasNext())
{
app.WaitForState(taskIter.Next(), TaskState.Running);
}
// Process the update events
clock.SetTime(Runtime.CurrentTimeMillis() + 1000);
EventHandler appEventHandler = app.GetContext().GetEventHandler();
foreach (KeyValuePair <TaskId, Task> mapTask in tasks)
{
foreach (KeyValuePair <TaskAttemptId, TaskAttempt> taskAttempt in mapTask.Value.GetAttempts
())
{
TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = CreateTaskAttemptStatus(taskAttempt
.Key, (float)0.5, TaskAttemptState.Running);
TaskAttemptStatusUpdateEvent @event = new TaskAttemptStatusUpdateEvent(taskAttempt
.Key, status);
appEventHandler.Handle(@event);
}
}
Task speculatedTask = null;
int numTasksToFinish = NumMappers + NumReducers - 1;
clock.SetTime(Runtime.CurrentTimeMillis() + 1000);
foreach (KeyValuePair <TaskId, Task> task in tasks)
{
foreach (KeyValuePair <TaskAttemptId, TaskAttempt> taskAttempt in task.Value.GetAttempts
())
{
if (numTasksToFinish > 0)
{
appEventHandler.Handle(new TaskAttemptEvent(taskAttempt.Key, TaskAttemptEventType
.TaDone));
appEventHandler.Handle(new TaskAttemptEvent(taskAttempt.Key, TaskAttemptEventType
.TaContainerCleaned));
numTasksToFinish--;
app.WaitForState(taskAttempt.Value, TaskAttemptState.Succeeded);
}
else
{
// The last task is chosen for speculation
TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = CreateTaskAttemptStatus(taskAttempt
.Key, (float)0.75, TaskAttemptState.Running);
speculatedTask = task.Value;
TaskAttemptStatusUpdateEvent @event = new TaskAttemptStatusUpdateEvent(taskAttempt
.Key, status);
appEventHandler.Handle(@event);
}
}
}
clock.SetTime(Runtime.CurrentTimeMillis() + 15000);
foreach (KeyValuePair <TaskId, Task> task_1 in tasks)
{
foreach (KeyValuePair <TaskAttemptId, TaskAttempt> taskAttempt in task_1.Value.GetAttempts
())
{
if (taskAttempt.Value.GetState() != TaskAttemptState.Succeeded)
{
TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = CreateTaskAttemptStatus(taskAttempt
.Key, (float)0.75, TaskAttemptState.Running);
TaskAttemptStatusUpdateEvent @event = new TaskAttemptStatusUpdateEvent(taskAttempt
.Key, status);
appEventHandler.Handle(@event);
}
}
}
Task speculatedTaskConst = speculatedTask;
GenericTestUtils.WaitFor(new _Supplier_205(speculatedTaskConst, clock), 1000, 60000
);
TaskAttempt[] ta = MakeFirstAttemptWin(appEventHandler, speculatedTask);
VerifySpeculationMessage(app, ta);
app.WaitForState(Service.STATE.Stopped);
}
/// <exception cref="System.Exception"/>
private void CheckHistoryParsing(int numMaps, int numReduces, int numSuccessfulMaps
)
{
Configuration conf = new Configuration();
conf.Set(MRJobConfig.UserName, Runtime.GetProperty("user.name"));
long amStartTimeEst = Runtime.CurrentTimeMillis();
conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping));
RackResolver.Init(conf);
MRApp app = new TestJobHistoryEvents.MRAppWithHistory(numMaps, numReduces, true,
this.GetType().FullName, true);
app.Submit(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
JobId jobId = job.GetID();
Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString());
app.WaitForState(job, JobState.Succeeded);
// make sure all events are flushed
app.WaitForState(Service.STATE.Stopped);
string jobhistoryDir = JobHistoryUtils.GetHistoryIntermediateDoneDirForUser(conf);
FileContext fc = null;
try
{
fc = FileContext.GetFileContext(conf);
}
catch (IOException ioe)
{
Log.Info("Can not get FileContext", ioe);
throw (new Exception("Can not get File Context"));
}
if (numMaps == numSuccessfulMaps)
{
string summaryFileName = JobHistoryUtils.GetIntermediateSummaryFileName(jobId);
Path summaryFile = new Path(jobhistoryDir, summaryFileName);
string jobSummaryString = GetJobSummary(fc, summaryFile);
NUnit.Framework.Assert.IsNotNull(jobSummaryString);
NUnit.Framework.Assert.IsTrue(jobSummaryString.Contains("resourcesPerMap=100"));
NUnit.Framework.Assert.IsTrue(jobSummaryString.Contains("resourcesPerReduce=100")
);
IDictionary <string, string> jobSummaryElements = new Dictionary <string, string>();
StringTokenizer strToken = new StringTokenizer(jobSummaryString, ",");
while (strToken.HasMoreTokens())
{
string keypair = strToken.NextToken();
jobSummaryElements[keypair.Split("=")[0]] = keypair.Split("=")[1];
}
NUnit.Framework.Assert.AreEqual("JobId does not match", jobId.ToString(), jobSummaryElements
["jobId"]);
NUnit.Framework.Assert.AreEqual("JobName does not match", "test", jobSummaryElements
["jobName"]);
NUnit.Framework.Assert.IsTrue("submitTime should not be 0", long.Parse(jobSummaryElements
["submitTime"]) != 0);
NUnit.Framework.Assert.IsTrue("launchTime should not be 0", long.Parse(jobSummaryElements
["launchTime"]) != 0);
NUnit.Framework.Assert.IsTrue("firstMapTaskLaunchTime should not be 0", long.Parse
(jobSummaryElements["firstMapTaskLaunchTime"]) != 0);
NUnit.Framework.Assert.IsTrue("firstReduceTaskLaunchTime should not be 0", long.Parse
(jobSummaryElements["firstReduceTaskLaunchTime"]) != 0);
NUnit.Framework.Assert.IsTrue("finishTime should not be 0", long.Parse(jobSummaryElements
["finishTime"]) != 0);
NUnit.Framework.Assert.AreEqual("Mismatch in num map slots", numSuccessfulMaps, System.Convert.ToInt32
(jobSummaryElements["numMaps"]));
NUnit.Framework.Assert.AreEqual("Mismatch in num reduce slots", numReduces, System.Convert.ToInt32
(jobSummaryElements["numReduces"]));
NUnit.Framework.Assert.AreEqual("User does not match", Runtime.GetProperty("user.name"
), jobSummaryElements["user"]);
NUnit.Framework.Assert.AreEqual("Queue does not match", "default", jobSummaryElements
["queue"]);
NUnit.Framework.Assert.AreEqual("Status does not match", "SUCCEEDED", jobSummaryElements
["status"]);
}
JobHistory jobHistory = new JobHistory();
jobHistory.Init(conf);
HistoryFileManager.HistoryFileInfo fileInfo = jobHistory.GetJobFileInfo(jobId);
JobHistoryParser.JobInfo jobInfo;
long numFinishedMaps;
lock (fileInfo)
{
Path historyFilePath = fileInfo.GetHistoryFile();
FSDataInputStream @in = null;
Log.Info("JobHistoryFile is: " + historyFilePath);
try
{
@in = fc.Open(fc.MakeQualified(historyFilePath));
}
catch (IOException ioe)
{
Log.Info("Can not open history file: " + historyFilePath, ioe);
throw (new Exception("Can not open History File"));
}
JobHistoryParser parser = new JobHistoryParser(@in);
EventReader realReader = new EventReader(@in);
EventReader reader = Org.Mockito.Mockito.Mock <EventReader>();
if (numMaps == numSuccessfulMaps)
{
reader = realReader;
}
else
{
AtomicInteger numFinishedEvents = new AtomicInteger(0);
// Hack!
Org.Mockito.Mockito.When(reader.GetNextEvent()).ThenAnswer(new _Answer_257(realReader
, numFinishedEvents, numSuccessfulMaps));
}
jobInfo = parser.Parse(reader);
numFinishedMaps = ComputeFinishedMaps(jobInfo, numMaps, numSuccessfulMaps);
if (numFinishedMaps != numMaps)
{
Exception parseException = parser.GetParseException();
NUnit.Framework.Assert.IsNotNull("Didn't get expected parse exception", parseException
);
}
}
NUnit.Framework.Assert.AreEqual("Incorrect username ", Runtime.GetProperty("user.name"
), jobInfo.GetUsername());
NUnit.Framework.Assert.AreEqual("Incorrect jobName ", "test", jobInfo.GetJobname(
));
NUnit.Framework.Assert.AreEqual("Incorrect queuename ", "default", jobInfo.GetJobQueueName
());
NUnit.Framework.Assert.AreEqual("incorrect conf path", "test", jobInfo.GetJobConfPath
());
NUnit.Framework.Assert.AreEqual("incorrect finishedMap ", numSuccessfulMaps, numFinishedMaps
);
NUnit.Framework.Assert.AreEqual("incorrect finishedReduces ", numReduces, jobInfo
.GetFinishedReduces());
NUnit.Framework.Assert.AreEqual("incorrect uberized ", job.IsUber(), jobInfo.GetUberized
());
IDictionary <TaskID, JobHistoryParser.TaskInfo> allTasks = jobInfo.GetAllTasks();
int totalTasks = allTasks.Count;
NUnit.Framework.Assert.AreEqual("total number of tasks is incorrect ", (numMaps
+ numReduces), totalTasks);
// Verify aminfo
NUnit.Framework.Assert.AreEqual(1, jobInfo.GetAMInfos().Count);
NUnit.Framework.Assert.AreEqual(MRApp.NmHost, jobInfo.GetAMInfos()[0].GetNodeManagerHost
());
JobHistoryParser.AMInfo amInfo = jobInfo.GetAMInfos()[0];
NUnit.Framework.Assert.AreEqual(MRApp.NmPort, amInfo.GetNodeManagerPort());
NUnit.Framework.Assert.AreEqual(MRApp.NmHttpPort, amInfo.GetNodeManagerHttpPort()
);
NUnit.Framework.Assert.AreEqual(1, amInfo.GetAppAttemptId().GetAttemptId());
NUnit.Framework.Assert.AreEqual(amInfo.GetAppAttemptId(), amInfo.GetContainerId()
.GetApplicationAttemptId());
NUnit.Framework.Assert.IsTrue(amInfo.GetStartTime() <= Runtime.CurrentTimeMillis(
) && amInfo.GetStartTime() >= amStartTimeEst);
ContainerId fakeCid = MRApp.NewContainerId(-1, -1, -1, -1);
// Assert at taskAttempt level
foreach (JobHistoryParser.TaskInfo taskInfo in allTasks.Values)
{
int taskAttemptCount = taskInfo.GetAllTaskAttempts().Count;
NUnit.Framework.Assert.AreEqual("total number of task attempts ", 1, taskAttemptCount
);
JobHistoryParser.TaskAttemptInfo taInfo = taskInfo.GetAllTaskAttempts().Values.GetEnumerator
().Next();
NUnit.Framework.Assert.IsNotNull(taInfo.GetContainerId());
// Verify the wrong ctor is not being used. Remove after mrv1 is removed.
NUnit.Framework.Assert.IsFalse(taInfo.GetContainerId().Equals(fakeCid));
}
// Deep compare Job and JobInfo
foreach (Task task in job.GetTasks().Values)
{
JobHistoryParser.TaskInfo taskInfo_1 = allTasks[TypeConverter.FromYarn(task.GetID
())];
NUnit.Framework.Assert.IsNotNull("TaskInfo not found", taskInfo_1);
foreach (TaskAttempt taskAttempt in task.GetAttempts().Values)
{
JobHistoryParser.TaskAttemptInfo taskAttemptInfo = taskInfo_1.GetAllTaskAttempts(
)[TypeConverter.FromYarn((taskAttempt.GetID()))];
NUnit.Framework.Assert.IsNotNull("TaskAttemptInfo not found", taskAttemptInfo);
NUnit.Framework.Assert.AreEqual("Incorrect shuffle port for task attempt", taskAttempt
.GetShufflePort(), taskAttemptInfo.GetShufflePort());
if (numMaps == numSuccessfulMaps)
{
NUnit.Framework.Assert.AreEqual(MRApp.NmHost, taskAttemptInfo.GetHostname());
NUnit.Framework.Assert.AreEqual(MRApp.NmPort, taskAttemptInfo.GetPort());
// Verify rack-name
NUnit.Framework.Assert.AreEqual("rack-name is incorrect", taskAttemptInfo.GetRackname
(), RackName);
}
}
}
// test output for HistoryViewer
TextWriter stdps = System.Console.Out;
try
{
Runtime.SetOut(new TextWriter(outContent));
HistoryViewer viewer;
lock (fileInfo)
{
viewer = new HistoryViewer(fc.MakeQualified(fileInfo.GetHistoryFile()).ToString()
, conf, true);
}
viewer.Print();
foreach (JobHistoryParser.TaskInfo taskInfo_1 in allTasks.Values)
{
string test = (taskInfo_1.GetTaskStatus() == null ? string.Empty : taskInfo_1.GetTaskStatus
()) + " " + taskInfo_1.GetTaskType() + " task list for " + taskInfo_1.GetTaskId(
).GetJobID();
NUnit.Framework.Assert.IsTrue(outContent.ToString().IndexOf(test) > 0);
NUnit.Framework.Assert.IsTrue(outContent.ToString().IndexOf(taskInfo_1.GetTaskId(
).ToString()) > 0);
}
}
finally
{
Runtime.SetOut(stdps);
}
}