public virtual void TestCommitPending()
{
MRApp app = new MRApp(1, 0, false, this.GetType().FullName, true);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration());
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task task = it.Next();
app.WaitForState(task, TaskState.Running);
TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next();
app.WaitForState(attempt, TaskAttemptState.Running);
//send the commit pending signal to the task
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(attempt.GetID(), TaskAttemptEventType
.TaCommitPending));
//wait for first attempt to commit pending
app.WaitForState(attempt, TaskAttemptState.CommitPending);
//re-send the commit pending signal to the task
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(attempt.GetID(), TaskAttemptEventType
.TaCommitPending));
//the task attempt should be still at COMMIT_PENDING
app.WaitForState(attempt, TaskAttemptState.CommitPending);
//send the done signal to the task
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task.GetAttempts()
.Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone));
app.WaitForState(job, JobState.Succeeded);
}
public virtual void VerifyHsTaskCountersXML(NodeList nodes, TaskAttempt att)
{
for (int i = 0; i < nodes.GetLength(); i++)
{
Element element = (Element)nodes.Item(i);
WebServicesTestUtils.CheckStringMatch("id", MRApps.ToString(att.GetID()), WebServicesTestUtils
.GetXmlString(element, "id"));
// just do simple verification of fields - not data is correct
// in the fields
NodeList groups = element.GetElementsByTagName("taskAttemptCounterGroup");
for (int j = 0; j < groups.GetLength(); j++)
{
Element counters = (Element)groups.Item(j);
NUnit.Framework.Assert.IsNotNull("should have counters in the web service info",
counters);
string name = WebServicesTestUtils.GetXmlString(counters, "counterGroupName");
NUnit.Framework.Assert.IsTrue("name not set", (name != null && !name.IsEmpty()));
NodeList counterArr = counters.GetElementsByTagName("counter");
for (int z = 0; z < counterArr.GetLength(); z++)
{
Element counter = (Element)counterArr.Item(z);
string counterName = WebServicesTestUtils.GetXmlString(counter, "name");
NUnit.Framework.Assert.IsTrue("counter name not set", (counterName != null && !counterName
.IsEmpty()));
long value = WebServicesTestUtils.GetXmlLong(counter, "value");
NUnit.Framework.Assert.IsTrue("value not >= 0", value >= 0);
}
}
}
}
public TaskInfo(Task task)
{
TaskType ttype = task.GetType();
this.type = ttype.ToString();
TaskReport report = task.GetReport();
this.startTime = report.GetStartTime();
this.finishTime = report.GetFinishTime();
this.state = report.GetTaskState();
this.elapsedTime = Times.Elapsed(this.startTime, this.finishTime, this.state == TaskState
.Running);
if (this.elapsedTime == -1)
{
this.elapsedTime = 0;
}
this.progress = report.GetProgress() * 100;
this.status = report.GetStatus();
this.id = MRApps.ToString(task.GetID());
this.taskNum = task.GetID().GetId();
this.successful = GetSuccessfulAttempt(task);
if (successful != null)
{
this.successfulAttempt = MRApps.ToString(successful.GetID());
}
else
{
this.successfulAttempt = string.Empty;
}
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
public virtual void VerifyHsJobTaskAttemptCounters(JSONObject info, TaskAttempt att
)
{
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 2, info.Length());
WebServicesTestUtils.CheckStringMatch("id", MRApps.ToString(att.GetID()), info.GetString
("id"));
// just do simple verification of fields - not data is correct
// in the fields
JSONArray counterGroups = info.GetJSONArray("taskAttemptCounterGroup");
for (int i = 0; i < counterGroups.Length(); i++)
{
JSONObject counterGroup = counterGroups.GetJSONObject(i);
string name = counterGroup.GetString("counterGroupName");
NUnit.Framework.Assert.IsTrue("name not set", (name != null && !name.IsEmpty()));
JSONArray counters = counterGroup.GetJSONArray("counter");
for (int j = 0; j < counters.Length(); j++)
{
JSONObject counter = counters.GetJSONObject(j);
string counterName = counter.GetString("name");
NUnit.Framework.Assert.IsTrue("name not set", (counterName != null && !counterName
.IsEmpty()));
long value = counter.GetLong("value");
NUnit.Framework.Assert.IsTrue("value >= 0", value >= 0);
}
}
}
public virtual void TestTaskFailWithUnusedContainer()
{
MRApp app = new TestFail.MRAppWithFailingTaskAndUnusedContainer();
Configuration conf = new Configuration();
int maxAttempts = 1;
conf.SetInt(MRJobConfig.MapMaxAttempts, maxAttempts);
// disable uberization (requires entire job to be reattempted, so max for
// subtask attempts is overridden to 1)
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
IDictionary <TaskId, Task> tasks = job.GetTasks();
NUnit.Framework.Assert.AreEqual("Num tasks is not correct", 1, tasks.Count);
Task task = tasks.Values.GetEnumerator().Next();
app.WaitForState(task, TaskState.Scheduled);
IDictionary <TaskAttemptId, TaskAttempt> attempts = tasks.Values.GetEnumerator().Next
().GetAttempts();
NUnit.Framework.Assert.AreEqual("Num attempts is not correct", maxAttempts, attempts
.Count);
TaskAttempt attempt = attempts.Values.GetEnumerator().Next();
app.WaitForInternalState((TaskAttemptImpl)attempt, TaskAttemptStateInternal.Assigned
);
app.GetDispatcher().GetEventHandler().Handle(new TaskAttemptEvent(attempt.GetID()
, TaskAttemptEventType.TaContainerCompleted));
app.WaitForState(job, JobState.Failed);
}
private void UpdateStatus(MRApp app, TaskAttempt attempt, Phase phase)
{
TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus
();
status.counters = new Counters();
status.fetchFailedMaps = new AList <TaskAttemptId>();
status.id = attempt.GetID();
status.mapFinishTime = 0;
status.phase = phase;
status.progress = 0.5f;
status.shuffleFinishTime = 0;
status.sortFinishTime = 0;
status.stateString = "OK";
status.taskState = attempt.GetState();
TaskAttemptStatusUpdateEvent @event = new TaskAttemptStatusUpdateEvent(attempt.GetID
(), status);
app.GetContext().GetEventHandler().Handle(@event);
}
public static IDictionary <TaskAttemptId, TaskAttempt> NewTaskAttempts(TaskId tid,
int m)
{
IDictionary <TaskAttemptId, TaskAttempt> map = Maps.NewHashMap();
for (int i = 0; i < m; ++i)
{
TaskAttempt ta = NewTaskAttempt(tid, i);
map[ta.GetID()] = ta;
}
return(map);
}
public virtual void TestKillTaskAttempt()
{
CountDownLatch latch = new CountDownLatch(1);
MRApp app = new TestKill.BlockingMRApp(2, 0, latch);
//this will start the job but job won't complete as Task is blocked
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration());
//wait and vailidate for Job to become RUNNING
app.WaitForState(job, JobState.Running);
IDictionary <TaskId, Task> tasks = job.GetTasks();
NUnit.Framework.Assert.AreEqual("No of tasks is not correct", 2, tasks.Count);
IEnumerator <Task> it = tasks.Values.GetEnumerator();
Task task1 = it.Next();
Task task2 = it.Next();
//wait for tasks to become running
app.WaitForState(task1, TaskState.Scheduled);
app.WaitForState(task2, TaskState.Scheduled);
//send the kill signal to the first Task's attempt
TaskAttempt attempt = task1.GetAttempts().Values.GetEnumerator().Next();
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(attempt.GetID(), TaskAttemptEventType
.TaKill));
//unblock
latch.CountDown();
//wait and validate for Job to become SUCCEEDED
//job will still succeed
app.WaitForState(job, JobState.Succeeded);
//first Task will have two attempts 1st is killed, 2nd Succeeds
//both Tasks and Job succeeds
NUnit.Framework.Assert.AreEqual("Task state not correct", TaskState.Succeeded, task1
.GetReport().GetTaskState());
NUnit.Framework.Assert.AreEqual("Task state not correct", TaskState.Succeeded, task2
.GetReport().GetTaskState());
IDictionary <TaskAttemptId, TaskAttempt> attempts = task1.GetAttempts();
NUnit.Framework.Assert.AreEqual("No of attempts is not correct", 2, attempts.Count
);
IEnumerator <TaskAttempt> iter = attempts.Values.GetEnumerator();
NUnit.Framework.Assert.AreEqual("Attempt state not correct", TaskAttemptState.Killed
, iter.Next().GetReport().GetTaskAttemptState());
NUnit.Framework.Assert.AreEqual("Attempt state not correct", TaskAttemptState.Succeeded
, iter.Next().GetReport().GetTaskAttemptState());
attempts = task2.GetAttempts();
NUnit.Framework.Assert.AreEqual("No of attempts is not correct", 1, attempts.Count
);
iter = attempts.Values.GetEnumerator();
NUnit.Framework.Assert.AreEqual("Attempt state not correct", TaskAttemptState.Succeeded
, iter.Next().GetReport().GetTaskAttemptState());
}
public virtual void TestKillTaskWaitKillJobAfterTA_DONE()
{
CountDownLatch latch = new CountDownLatch(1);
Dispatcher dispatcher = new TestKill.MyAsyncDispatch(latch, TaskAttemptEventType.
TaDone);
MRApp app = new _MRApp_252(dispatcher, 1, 1, false, this.GetType().FullName, true
);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration());
JobId jobId = app.GetJobId();
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 2, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task mapTask = it.Next();
Task reduceTask = it.Next();
app.WaitForState(mapTask, TaskState.Running);
app.WaitForState(reduceTask, TaskState.Running);
TaskAttempt mapAttempt = mapTask.GetAttempts().Values.GetEnumerator().Next();
app.WaitForState(mapAttempt, TaskAttemptState.Running);
TaskAttempt reduceAttempt = reduceTask.GetAttempts().Values.GetEnumerator().Next(
);
app.WaitForState(reduceAttempt, TaskAttemptState.Running);
// The order in the dispatch event queue, from the oldest to the newest
// TA_DONE
// JOB_KILL
// CONTAINER_REMOTE_CLEANUP ( from TA_DONE's handling )
// T_KILL ( from JOB_KILL's handling )
// TA_CONTAINER_CLEANED ( from CONTAINER_REMOTE_CLEANUP's handling )
// TA_KILL ( from T_KILL's handling )
// T_ATTEMPT_SUCCEEDED ( from TA_CONTAINER_CLEANED's handling )
// T_ATTEMPT_KILLED ( from TA_KILL's handling )
// Finish map
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapAttempt.GetID()
, TaskAttemptEventType.TaDone));
// Now kill the job
app.GetContext().GetEventHandler().Handle(new JobEvent(jobId, JobEventType.JobKill
));
//unblock
latch.CountDown();
app.WaitForInternalState((JobImpl)job, JobStateInternal.Killed);
}
public JobTaskAttemptCounterInfo(TaskAttempt taskattempt)
{
this.id = MRApps.ToString(taskattempt.GetID());
total = taskattempt.GetCounters();
taskAttemptCounterGroup = new AList <TaskCounterGroupInfo>();
if (total != null)
{
foreach (CounterGroup g in total)
{
if (g != null)
{
TaskCounterGroupInfo cginfo = new TaskCounterGroupInfo(g.GetName(), g);
if (cginfo != null)
{
taskAttemptCounterGroup.AddItem(cginfo);
}
}
}
}
}
public TaskAttemptInfo(TaskAttempt ta, TaskType type, bool isRunning)
{
TaskAttemptReport report = ta.GetReport();
this.type = type.ToString();
this.id = MRApps.ToString(ta.GetID());
this.nodeHttpAddress = ta.GetNodeHttpAddress();
this.startTime = report.GetStartTime();
this.finishTime = report.GetFinishTime();
this.assignedContainerId = ConverterUtils.ToString(report.GetContainerId());
this.assignedContainer = report.GetContainerId();
this.progress = report.GetProgress() * 100;
this.status = report.GetStateString();
this.state = report.GetTaskAttemptState();
this.elapsedTime = Times.Elapsed(this.startTime, this.finishTime, isRunning);
if (this.elapsedTime == -1)
{
this.elapsedTime = 0;
}
this.diagnostics = report.GetDiagnosticInfo();
this.rack = ta.GetNodeRackName();
}
public virtual void TestKillTaskWait()
{
Dispatcher dispatcher = new _AsyncDispatcher_145();
// Task is asking the reduce TA to kill itself. 'Create' a race
// condition. Make the task succeed and then inform the task that
// TA has succeeded. Once Task gets the TA succeeded event at
// KILL_WAIT, then relay the actual kill signal to TA
// When the TA comes and reports that it is done, send the
// cachedKillEvent
MRApp app = new _MRApp_183(dispatcher, 1, 1, false, this.GetType().FullName, true
);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration());
JobId jobId = app.GetJobId();
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 2, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task mapTask = it.Next();
Task reduceTask = it.Next();
app.WaitForState(mapTask, TaskState.Running);
app.WaitForState(reduceTask, TaskState.Running);
TaskAttempt mapAttempt = mapTask.GetAttempts().Values.GetEnumerator().Next();
app.WaitForState(mapAttempt, TaskAttemptState.Running);
TaskAttempt reduceAttempt = reduceTask.GetAttempts().Values.GetEnumerator().Next(
);
app.WaitForState(reduceAttempt, TaskAttemptState.Running);
// Finish map
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapAttempt.GetID()
, TaskAttemptEventType.TaDone));
app.WaitForState(mapTask, TaskState.Succeeded);
// Now kill the job
app.GetContext().GetEventHandler().Handle(new JobEvent(jobId, JobEventType.JobKill
));
app.WaitForInternalState((JobImpl)job, JobStateInternal.Killed);
}
public virtual void VerifyTaskAttemptGeneric(TaskAttempt ta, TaskType ttype, string
id, string state, string type, string rack, string nodeHttpAddress, string diagnostics
, string assignedContainerId, long startTime, long finishTime, long elapsedTime,
float progress)
{
TaskAttemptId attid = ta.GetID();
string attemptId = MRApps.ToString(attid);
WebServicesTestUtils.CheckStringMatch("id", attemptId, id);
WebServicesTestUtils.CheckStringMatch("type", ttype.ToString(), type);
WebServicesTestUtils.CheckStringMatch("state", ta.GetState().ToString(), state);
WebServicesTestUtils.CheckStringMatch("rack", ta.GetNodeRackName(), rack);
WebServicesTestUtils.CheckStringMatch("nodeHttpAddress", ta.GetNodeHttpAddress(),
nodeHttpAddress);
string expectDiag = string.Empty;
IList <string> diagnosticsList = ta.GetDiagnostics();
if (diagnosticsList != null && !diagnostics.IsEmpty())
{
StringBuilder b = new StringBuilder();
foreach (string diag in diagnosticsList)
{
b.Append(diag);
}
expectDiag = b.ToString();
}
WebServicesTestUtils.CheckStringMatch("diagnostics", expectDiag, diagnostics);
WebServicesTestUtils.CheckStringMatch("assignedContainerId", ConverterUtils.ToString
(ta.GetAssignedContainerID()), assignedContainerId);
NUnit.Framework.Assert.AreEqual("startTime wrong", ta.GetLaunchTime(), startTime);
NUnit.Framework.Assert.AreEqual("finishTime wrong", ta.GetFinishTime(), finishTime
);
NUnit.Framework.Assert.AreEqual("elapsedTime wrong", finishTime - startTime, elapsedTime
);
NUnit.Framework.Assert.AreEqual("progress wrong", ta.GetProgress() * 100, progress
, 1e-3f);
}
public virtual void TestFetchFailureMultipleReduces()
{
MRApp app = new MRApp(1, 3, false, this.GetType().FullName, true);
Configuration conf = new Configuration();
// map -> reduce -> fetch-failure -> map retry is incompatible with
// sequential, single-task-attempt approach in uber-AM, so disable:
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
//all maps would be running
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 4, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task mapTask = it.Next();
Task reduceTask = it.Next();
Task reduceTask2 = it.Next();
Task reduceTask3 = it.Next();
//wait for Task state move to RUNNING
app.WaitForState(mapTask, TaskState.Running);
TaskAttempt mapAttempt1 = mapTask.GetAttempts().Values.GetEnumerator().Next();
app.WaitForState(mapAttempt1, TaskAttemptState.Running);
//send the done signal to the map attempt
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapAttempt1.GetID(
), TaskAttemptEventType.TaDone));
// wait for map success
app.WaitForState(mapTask, TaskState.Succeeded);
TaskAttemptCompletionEvent[] events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Num completion events not correct", 1, events.Length
);
NUnit.Framework.Assert.AreEqual("Event status not correct", TaskAttemptCompletionEventStatus
.Succeeded, events[0].GetStatus());
// wait for reduce to start running
app.WaitForState(reduceTask, TaskState.Running);
app.WaitForState(reduceTask2, TaskState.Running);
app.WaitForState(reduceTask3, TaskState.Running);
TaskAttempt reduceAttempt = reduceTask.GetAttempts().Values.GetEnumerator().Next(
);
app.WaitForState(reduceAttempt, TaskAttemptState.Running);
UpdateStatus(app, reduceAttempt, Phase.Shuffle);
TaskAttempt reduceAttempt2 = reduceTask2.GetAttempts().Values.GetEnumerator().Next
();
app.WaitForState(reduceAttempt2, TaskAttemptState.Running);
UpdateStatus(app, reduceAttempt2, Phase.Shuffle);
TaskAttempt reduceAttempt3 = reduceTask3.GetAttempts().Values.GetEnumerator().Next
();
app.WaitForState(reduceAttempt3, TaskAttemptState.Running);
UpdateStatus(app, reduceAttempt3, Phase.Shuffle);
//send 2 fetch failures from reduce to prepare for map re execution
SendFetchFailure(app, reduceAttempt, mapAttempt1);
SendFetchFailure(app, reduceAttempt, mapAttempt1);
//We should not re-launch the map task yet
NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, mapTask.GetState());
UpdateStatus(app, reduceAttempt2, Phase.Reduce);
UpdateStatus(app, reduceAttempt3, Phase.Reduce);
//send 3rd fetch failures from reduce to trigger map re execution
SendFetchFailure(app, reduceAttempt, mapAttempt1);
//wait for map Task state move back to RUNNING
app.WaitForState(mapTask, TaskState.Running);
//map attempt must have become FAILED
NUnit.Framework.Assert.AreEqual("Map TaskAttempt state not correct", TaskAttemptState
.Failed, mapAttempt1.GetState());
NUnit.Framework.Assert.AreEqual("Num attempts in Map Task not correct", 2, mapTask
.GetAttempts().Count);
IEnumerator <TaskAttempt> atIt = mapTask.GetAttempts().Values.GetEnumerator();
atIt.Next();
TaskAttempt mapAttempt2 = atIt.Next();
app.WaitForState(mapAttempt2, TaskAttemptState.Running);
//send the done signal to the second map attempt
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapAttempt2.GetID(
), TaskAttemptEventType.TaDone));
// wait for map success
app.WaitForState(mapTask, TaskState.Succeeded);
//send done to reduce
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(reduceAttempt.GetID
(), TaskAttemptEventType.TaDone));
//send done to reduce
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(reduceAttempt2.GetID
(), TaskAttemptEventType.TaDone));
//send done to reduce
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(reduceAttempt3.GetID
(), TaskAttemptEventType.TaDone));
app.WaitForState(job, JobState.Succeeded);
//previous completion event now becomes obsolete
NUnit.Framework.Assert.AreEqual("Event status not correct", TaskAttemptCompletionEventStatus
.Obsolete, events[0].GetStatus());
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Num completion events not correct", 6, events.Length
);
NUnit.Framework.Assert.AreEqual("Event map attempt id not correct", mapAttempt1.GetID
(), events[0].GetAttemptId());
NUnit.Framework.Assert.AreEqual("Event map attempt id not correct", mapAttempt1.GetID
(), events[1].GetAttemptId());
NUnit.Framework.Assert.AreEqual("Event map attempt id not correct", mapAttempt2.GetID
(), events[2].GetAttemptId());
NUnit.Framework.Assert.AreEqual("Event reduce attempt id not correct", reduceAttempt
.GetID(), events[3].GetAttemptId());
NUnit.Framework.Assert.AreEqual("Event status not correct for map attempt1", TaskAttemptCompletionEventStatus
.Obsolete, events[0].GetStatus());
NUnit.Framework.Assert.AreEqual("Event status not correct for map attempt1", TaskAttemptCompletionEventStatus
.Failed, events[1].GetStatus());
NUnit.Framework.Assert.AreEqual("Event status not correct for map attempt2", TaskAttemptCompletionEventStatus
.Succeeded, events[2].GetStatus());
NUnit.Framework.Assert.AreEqual("Event status not correct for reduce attempt1", TaskAttemptCompletionEventStatus
.Succeeded, events[3].GetStatus());
TaskCompletionEvent[] mapEvents = job.GetMapAttemptCompletionEvents(0, 2);
TaskCompletionEvent[] convertedEvents = TypeConverter.FromYarn(events);
NUnit.Framework.Assert.AreEqual("Incorrect number of map events", 2, mapEvents.Length
);
Assert.AssertArrayEquals("Unexpected map events", Arrays.CopyOfRange(convertedEvents
, 0, 2), mapEvents);
mapEvents = job.GetMapAttemptCompletionEvents(2, 200);
NUnit.Framework.Assert.AreEqual("Incorrect number of map events", 1, mapEvents.Length
);
NUnit.Framework.Assert.AreEqual("Unexpected map event", convertedEvents[2], mapEvents
[0]);
}
public virtual void TestFetchFailureWithRecovery()
{
int runCount = 0;
MRApp app = new TestFetchFailure.MRAppWithHistory(1, 1, false, this.GetType().FullName
, true, ++runCount);
Configuration conf = new Configuration();
// map -> reduce -> fetch-failure -> map retry is incompatible with
// sequential, single-task-attempt approach in uber-AM, so disable:
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
//all maps would be running
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 2, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task mapTask = it.Next();
Task reduceTask = it.Next();
//wait for Task state move to RUNNING
app.WaitForState(mapTask, TaskState.Running);
TaskAttempt mapAttempt1 = mapTask.GetAttempts().Values.GetEnumerator().Next();
app.WaitForState(mapAttempt1, TaskAttemptState.Running);
//send the done signal to the map attempt
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapAttempt1.GetID(
), TaskAttemptEventType.TaDone));
// wait for map success
app.WaitForState(mapTask, TaskState.Succeeded);
TaskAttemptCompletionEvent[] events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Num completion events not correct", 1, events.Length
);
NUnit.Framework.Assert.AreEqual("Event status not correct", TaskAttemptCompletionEventStatus
.Succeeded, events[0].GetStatus());
// wait for reduce to start running
app.WaitForState(reduceTask, TaskState.Running);
TaskAttempt reduceAttempt = reduceTask.GetAttempts().Values.GetEnumerator().Next(
);
app.WaitForState(reduceAttempt, TaskAttemptState.Running);
//send 3 fetch failures from reduce to trigger map re execution
SendFetchFailure(app, reduceAttempt, mapAttempt1);
SendFetchFailure(app, reduceAttempt, mapAttempt1);
SendFetchFailure(app, reduceAttempt, mapAttempt1);
//wait for map Task state move back to RUNNING
app.WaitForState(mapTask, TaskState.Running);
// Crash the app again.
app.Stop();
//rerun
app = new TestFetchFailure.MRAppWithHistory(1, 1, false, this.GetType().FullName,
false, ++runCount);
conf = new Configuration();
conf.SetBoolean(MRJobConfig.MrAmJobRecoveryEnable, true);
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
//all maps would be running
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 2, job.GetTasks().Count);
it = job.GetTasks().Values.GetEnumerator();
mapTask = it.Next();
reduceTask = it.Next();
// the map is not in a SUCCEEDED state after restart of AM
app.WaitForState(mapTask, TaskState.Running);
mapAttempt1 = mapTask.GetAttempts().Values.GetEnumerator().Next();
app.WaitForState(mapAttempt1, TaskAttemptState.Running);
//send the done signal to the map attempt
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapAttempt1.GetID(
), TaskAttemptEventType.TaDone));
// wait for map success
app.WaitForState(mapTask, TaskState.Succeeded);
reduceAttempt = reduceTask.GetAttempts().Values.GetEnumerator().Next();
//send done to reduce
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(reduceAttempt.GetID
(), TaskAttemptEventType.TaDone));
app.WaitForState(job, JobState.Succeeded);
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Num completion events not correct", 2, events.Length
);
}
public virtual void Test()
{
TestMRClientService.MRAppWithClientService app = new TestMRClientService.MRAppWithClientService
(this, 1, 0, false);
Configuration conf = new Configuration();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task task = it.Next();
app.WaitForState(task, TaskState.Running);
TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next();
app.WaitForState(attempt, TaskAttemptState.Running);
// send the diagnostic
string diagnostic1 = "Diagnostic1";
string diagnostic2 = "Diagnostic2";
app.GetContext().GetEventHandler().Handle(new TaskAttemptDiagnosticsUpdateEvent(attempt
.GetID(), diagnostic1));
// send the status update
TaskAttemptStatusUpdateEvent.TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus
();
taskAttemptStatus.id = attempt.GetID();
taskAttemptStatus.progress = 0.5f;
taskAttemptStatus.stateString = "RUNNING";
taskAttemptStatus.taskState = TaskAttemptState.Running;
taskAttemptStatus.phase = Phase.Map;
// send the status update
app.GetContext().GetEventHandler().Handle(new TaskAttemptStatusUpdateEvent(attempt
.GetID(), taskAttemptStatus));
//verify that all object are fully populated by invoking RPCs.
YarnRPC rpc = YarnRPC.Create(conf);
MRClientProtocol proxy = (MRClientProtocol)rpc.GetProxy(typeof(MRClientProtocol),
app.clientService.GetBindAddress(), conf);
GetCountersRequest gcRequest = recordFactory.NewRecordInstance <GetCountersRequest
>();
gcRequest.SetJobId(job.GetID());
NUnit.Framework.Assert.IsNotNull("Counters is null", proxy.GetCounters(gcRequest)
.GetCounters());
GetJobReportRequest gjrRequest = recordFactory.NewRecordInstance <GetJobReportRequest
>();
gjrRequest.SetJobId(job.GetID());
JobReport jr = proxy.GetJobReport(gjrRequest).GetJobReport();
VerifyJobReport(jr);
GetTaskAttemptCompletionEventsRequest gtaceRequest = recordFactory.NewRecordInstance
<GetTaskAttemptCompletionEventsRequest>();
gtaceRequest.SetJobId(job.GetID());
gtaceRequest.SetFromEventId(0);
gtaceRequest.SetMaxEvents(10);
NUnit.Framework.Assert.IsNotNull("TaskCompletionEvents is null", proxy.GetTaskAttemptCompletionEvents
(gtaceRequest).GetCompletionEventList());
GetDiagnosticsRequest gdRequest = recordFactory.NewRecordInstance <GetDiagnosticsRequest
>();
gdRequest.SetTaskAttemptId(attempt.GetID());
NUnit.Framework.Assert.IsNotNull("Diagnostics is null", proxy.GetDiagnostics(gdRequest
).GetDiagnosticsList());
GetTaskAttemptReportRequest gtarRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest
>();
gtarRequest.SetTaskAttemptId(attempt.GetID());
TaskAttemptReport tar = proxy.GetTaskAttemptReport(gtarRequest).GetTaskAttemptReport
();
VerifyTaskAttemptReport(tar);
GetTaskReportRequest gtrRequest = recordFactory.NewRecordInstance <GetTaskReportRequest
>();
gtrRequest.SetTaskId(task.GetID());
NUnit.Framework.Assert.IsNotNull("TaskReport is null", proxy.GetTaskReport(gtrRequest
).GetTaskReport());
GetTaskReportsRequest gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest
>();
gtreportsRequest.SetJobId(job.GetID());
gtreportsRequest.SetTaskType(TaskType.Map);
NUnit.Framework.Assert.IsNotNull("TaskReports for map is null", proxy.GetTaskReports
(gtreportsRequest).GetTaskReportList());
gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>();
gtreportsRequest.SetJobId(job.GetID());
gtreportsRequest.SetTaskType(TaskType.Reduce);
NUnit.Framework.Assert.IsNotNull("TaskReports for reduce is null", proxy.GetTaskReports
(gtreportsRequest).GetTaskReportList());
IList <string> diag = proxy.GetDiagnostics(gdRequest).GetDiagnosticsList();
NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, diag.Count);
NUnit.Framework.Assert.AreEqual("Diag 1 not correct", diagnostic1, diag[0].ToString
());
TaskReport taskReport = proxy.GetTaskReport(gtrRequest).GetTaskReport();
NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, taskReport.GetDiagnosticsCount
());
//send the done signal to the task
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task.GetAttempts()
.Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone));
app.WaitForState(job, JobState.Succeeded);
// For invalid jobid, throw IOException
gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>();
gtreportsRequest.SetJobId(TypeConverter.ToYarn(JobID.ForName("job_1415730144495_0001"
)));
gtreportsRequest.SetTaskType(TaskType.Reduce);
try
{
proxy.GetTaskReports(gtreportsRequest);
NUnit.Framework.Assert.Fail("IOException not thrown for invalid job id");
}
catch (IOException)
{
}
}
public virtual void TestViewAclOnlyCannotModify()
{
TestMRClientService.MRAppWithClientService app = new TestMRClientService.MRAppWithClientService
(this, 1, 0, false);
Configuration conf = new Configuration();
conf.SetBoolean(MRConfig.MrAclsEnabled, true);
conf.Set(MRJobConfig.JobAclViewJob, "viewonlyuser");
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task task = it.Next();
app.WaitForState(task, TaskState.Running);
TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next();
app.WaitForState(attempt, TaskAttemptState.Running);
UserGroupInformation viewOnlyUser = UserGroupInformation.CreateUserForTesting("viewonlyuser"
, new string[] { });
NUnit.Framework.Assert.IsTrue("viewonlyuser cannot view job", job.CheckAccess(viewOnlyUser
, JobACL.ViewJob));
NUnit.Framework.Assert.IsFalse("viewonlyuser can modify job", job.CheckAccess(viewOnlyUser
, JobACL.ModifyJob));
MRClientProtocol client = viewOnlyUser.DoAs(new _PrivilegedExceptionAction_223(conf
, app));
KillJobRequest killJobRequest = recordFactory.NewRecordInstance <KillJobRequest>();
killJobRequest.SetJobId(app.GetJobId());
try
{
client.KillJob(killJobRequest);
NUnit.Framework.Assert.Fail("viewonlyuser killed job");
}
catch (AccessControlException)
{
}
// pass
KillTaskRequest killTaskRequest = recordFactory.NewRecordInstance <KillTaskRequest
>();
killTaskRequest.SetTaskId(task.GetID());
try
{
client.KillTask(killTaskRequest);
NUnit.Framework.Assert.Fail("viewonlyuser killed task");
}
catch (AccessControlException)
{
}
// pass
KillTaskAttemptRequest killTaskAttemptRequest = recordFactory.NewRecordInstance <KillTaskAttemptRequest
>();
killTaskAttemptRequest.SetTaskAttemptId(attempt.GetID());
try
{
client.KillTaskAttempt(killTaskAttemptRequest);
NUnit.Framework.Assert.Fail("viewonlyuser killed task attempt");
}
catch (AccessControlException)
{
}
// pass
FailTaskAttemptRequest failTaskAttemptRequest = recordFactory.NewRecordInstance <FailTaskAttemptRequest
>();
failTaskAttemptRequest.SetTaskAttemptId(attempt.GetID());
try
{
client.FailTaskAttempt(failTaskAttemptRequest);
NUnit.Framework.Assert.Fail("viewonlyuser killed task attempt");
}
catch (AccessControlException)
{
}
}
//Test reports of JobHistoryServer. History server should get log files from MRApp and read them
/// <exception cref="System.Exception"/>
public virtual void TestReports()
{
Configuration config = new Configuration();
config.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey
, typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping));
RackResolver.Init(config);
MRApp app = new TestJobHistoryEvents.MRAppWithHistory(1, 1, true, this.GetType().
FullName, true);
app.Submit(config);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
app.WaitForState(job, JobState.Succeeded);
historyServer = new JobHistoryServer();
historyServer.Init(config);
historyServer.Start();
// search JobHistory service
JobHistory jobHistory = null;
foreach (Org.Apache.Hadoop.Service.Service service in historyServer.GetServices())
{
if (service is JobHistory)
{
jobHistory = (JobHistory)service;
}
}
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobs = jobHistory.
GetAllJobs();
NUnit.Framework.Assert.AreEqual(1, jobs.Count);
NUnit.Framework.Assert.AreEqual("job_0_0000", jobs.Keys.GetEnumerator().Next().ToString
());
Task task = job.GetTasks().Values.GetEnumerator().Next();
TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next();
HistoryClientService historyService = historyServer.GetClientService();
MRClientProtocol protocol = historyService.GetClientHandler();
GetTaskAttemptReportRequest gtarRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest
>();
// test getTaskAttemptReport
TaskAttemptId taId = attempt.GetID();
taId.SetTaskId(task.GetID());
taId.GetTaskId().SetJobId(job.GetID());
gtarRequest.SetTaskAttemptId(taId);
GetTaskAttemptReportResponse response = protocol.GetTaskAttemptReport(gtarRequest
);
NUnit.Framework.Assert.AreEqual("container_0_0000_01_000000", response.GetTaskAttemptReport
().GetContainerId().ToString());
NUnit.Framework.Assert.IsTrue(response.GetTaskAttemptReport().GetDiagnosticInfo()
.IsEmpty());
// counters
NUnit.Framework.Assert.IsNotNull(response.GetTaskAttemptReport().GetCounters().GetCounter
(TaskCounter.PhysicalMemoryBytes));
NUnit.Framework.Assert.AreEqual(taId.ToString(), response.GetTaskAttemptReport().
GetTaskAttemptId().ToString());
// test getTaskReport
GetTaskReportRequest request = recordFactory.NewRecordInstance <GetTaskReportRequest
>();
TaskId taskId = task.GetID();
taskId.SetJobId(job.GetID());
request.SetTaskId(taskId);
GetTaskReportResponse reportResponse = protocol.GetTaskReport(request);
NUnit.Framework.Assert.AreEqual(string.Empty, reportResponse.GetTaskReport().GetDiagnosticsList
().GetEnumerator().Next());
// progress
NUnit.Framework.Assert.AreEqual(1.0f, reportResponse.GetTaskReport().GetProgress(
), 0.01);
// report has corrected taskId
NUnit.Framework.Assert.AreEqual(taskId.ToString(), reportResponse.GetTaskReport()
.GetTaskId().ToString());
// Task state should be SUCCEEDED
NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, reportResponse.GetTaskReport
().GetTaskState());
// For invalid jobid, throw IOException
GetTaskReportsRequest gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest
>();
gtreportsRequest.SetJobId(TypeConverter.ToYarn(JobID.ForName("job_1415730144495_0001"
)));
gtreportsRequest.SetTaskType(TaskType.Reduce);
try
{
protocol.GetTaskReports(gtreportsRequest);
NUnit.Framework.Assert.Fail("IOException not thrown for invalid job id");
}
catch (IOException)
{
}
// Expected
// test getTaskAttemptCompletionEvents
GetTaskAttemptCompletionEventsRequest taskAttemptRequest = recordFactory.NewRecordInstance
<GetTaskAttemptCompletionEventsRequest>();
taskAttemptRequest.SetJobId(job.GetID());
GetTaskAttemptCompletionEventsResponse taskAttemptCompletionEventsResponse = protocol
.GetTaskAttemptCompletionEvents(taskAttemptRequest);
NUnit.Framework.Assert.AreEqual(0, taskAttemptCompletionEventsResponse.GetCompletionEventCount
());
// test getDiagnostics
GetDiagnosticsRequest diagnosticRequest = recordFactory.NewRecordInstance <GetDiagnosticsRequest
>();
diagnosticRequest.SetTaskAttemptId(taId);
GetDiagnosticsResponse diagnosticResponse = protocol.GetDiagnostics(diagnosticRequest
);
// it is strange : why one empty string ?
NUnit.Framework.Assert.AreEqual(1, diagnosticResponse.GetDiagnosticsCount());
NUnit.Framework.Assert.AreEqual(string.Empty, diagnosticResponse.GetDiagnostics(0
));
}
public virtual void TestUpdatedNodes()
{
int runCount = 0;
MRApp app = new TestMRApp.MRAppWithHistory(this, 2, 2, false, this.GetType().FullName
, true, ++runCount);
Configuration conf = new Configuration();
// after half of the map completion, reduce will start
conf.SetFloat(MRJobConfig.CompletedMapsForReduceSlowstart, 0.5f);
// uberization forces full slowstart (1.0), so disable that
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 4, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task mapTask1 = it.Next();
Task mapTask2 = it.Next();
// all maps must be running
app.WaitForState(mapTask1, TaskState.Running);
app.WaitForState(mapTask2, TaskState.Running);
TaskAttempt task1Attempt = mapTask1.GetAttempts().Values.GetEnumerator().Next();
TaskAttempt task2Attempt = mapTask2.GetAttempts().Values.GetEnumerator().Next();
NodeId node1 = task1Attempt.GetNodeId();
NodeId node2 = task2Attempt.GetNodeId();
NUnit.Framework.Assert.AreEqual(node1, node2);
// send the done signal to the task
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task1Attempt.GetID
(), TaskAttemptEventType.TaDone));
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task2Attempt.GetID
(), TaskAttemptEventType.TaDone));
// all maps must be succeeded
app.WaitForState(mapTask1, TaskState.Succeeded);
app.WaitForState(mapTask2, TaskState.Succeeded);
TaskAttemptCompletionEvent[] events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 2 completion events for success", 2, events
.Length);
// send updated nodes info
AList <NodeReport> updatedNodes = new AList <NodeReport>();
NodeReport nr = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <NodeReport
>();
nr.SetNodeId(node1);
nr.SetNodeState(NodeState.Unhealthy);
updatedNodes.AddItem(nr);
app.GetContext().GetEventHandler().Handle(new JobUpdatedNodesEvent(job.GetID(), updatedNodes
));
app.WaitForState(task1Attempt, TaskAttemptState.Killed);
app.WaitForState(task2Attempt, TaskAttemptState.Killed);
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 2 more completion events for killed",
4, events.Length);
// all maps must be back to running
app.WaitForState(mapTask1, TaskState.Running);
app.WaitForState(mapTask2, TaskState.Running);
IEnumerator <TaskAttempt> itr = mapTask1.GetAttempts().Values.GetEnumerator();
itr.Next();
task1Attempt = itr.Next();
// send the done signal to the task
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task1Attempt.GetID
(), TaskAttemptEventType.TaDone));
// map1 must be succeeded. map2 must be running
app.WaitForState(mapTask1, TaskState.Succeeded);
app.WaitForState(mapTask2, TaskState.Running);
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 1 more completion events for success",
5, events.Length);
// Crash the app again.
app.Stop();
// rerun
// in rerun the 1st map will be recovered from previous run
app = new TestMRApp.MRAppWithHistory(this, 2, 2, false, this.GetType().FullName,
false, ++runCount);
conf = new Configuration();
conf.SetBoolean(MRJobConfig.MrAmJobRecoveryEnable, true);
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("No of tasks not correct", 4, job.GetTasks().Count
);
it = job.GetTasks().Values.GetEnumerator();
mapTask1 = it.Next();
mapTask2 = it.Next();
Task reduceTask1 = it.Next();
Task reduceTask2 = it.Next();
// map 1 will be recovered, no need to send done
app.WaitForState(mapTask1, TaskState.Succeeded);
app.WaitForState(mapTask2, TaskState.Running);
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 2 completion events for killed & success of map1"
, 2, events.Length);
task2Attempt = mapTask2.GetAttempts().Values.GetEnumerator().Next();
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task2Attempt.GetID
(), TaskAttemptEventType.TaDone));
app.WaitForState(mapTask2, TaskState.Succeeded);
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 1 more completion events for success",
3, events.Length);
app.WaitForState(reduceTask1, TaskState.Running);
app.WaitForState(reduceTask2, TaskState.Running);
TaskAttempt task3Attempt = reduceTask1.GetAttempts().Values.GetEnumerator().Next(
);
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task3Attempt.GetID
(), TaskAttemptEventType.TaDone));
app.WaitForState(reduceTask1, TaskState.Succeeded);
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task3Attempt.GetID
(), TaskAttemptEventType.TaKill));
app.WaitForState(reduceTask1, TaskState.Succeeded);
TaskAttempt task4Attempt = reduceTask2.GetAttempts().Values.GetEnumerator().Next(
);
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task4Attempt.GetID
(), TaskAttemptEventType.TaDone));
app.WaitForState(reduceTask2, TaskState.Succeeded);
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 2 more completion events for reduce success"
, 5, events.Length);
// job succeeds
app.WaitForState(job, JobState.Succeeded);
}
private void SendFetchFailure(MRApp app, TaskAttempt reduceAttempt, TaskAttempt mapAttempt
)
{
app.GetContext().GetEventHandler().Handle(new JobTaskAttemptFetchFailureEvent(reduceAttempt
.GetID(), Arrays.AsList(new TaskAttemptId[] { mapAttempt.GetID() })));
}
public virtual void TestAttemptsBlock()
{
AppContext ctx = Org.Mockito.Mockito.Mock <AppContext>();
AppForTest app = new AppForTest(ctx);
Task task = GetTask(0);
IDictionary <TaskAttemptId, TaskAttempt> attempts = new Dictionary <TaskAttemptId,
TaskAttempt>();
TaskAttempt attempt = Org.Mockito.Mockito.Mock <TaskAttempt>();
TaskAttemptId taId = new TaskAttemptIdPBImpl();
taId.SetId(0);
taId.SetTaskId(task.GetID());
Org.Mockito.Mockito.When(attempt.GetID()).ThenReturn(taId);
Org.Mockito.Mockito.When(attempt.GetNodeHttpAddress()).ThenReturn("Node address");
ApplicationId appId = ApplicationIdPBImpl.NewInstance(0, 5);
ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.NewInstance(appId,
1);
ContainerId containerId = ContainerIdPBImpl.NewContainerId(appAttemptId, 1);
Org.Mockito.Mockito.When(attempt.GetAssignedContainerID()).ThenReturn(containerId
);
Org.Mockito.Mockito.When(attempt.GetAssignedContainerMgrAddress()).ThenReturn("assignedContainerMgrAddress"
);
Org.Mockito.Mockito.When(attempt.GetNodeRackName()).ThenReturn("nodeRackName");
long taStartTime = 100002L;
long taFinishTime = 100012L;
long taShuffleFinishTime = 100010L;
long taSortFinishTime = 100011L;
TaskAttemptState taState = TaskAttemptState.Succeeded;
Org.Mockito.Mockito.When(attempt.GetLaunchTime()).ThenReturn(taStartTime);
Org.Mockito.Mockito.When(attempt.GetFinishTime()).ThenReturn(taFinishTime);
Org.Mockito.Mockito.When(attempt.GetShuffleFinishTime()).ThenReturn(taShuffleFinishTime
);
Org.Mockito.Mockito.When(attempt.GetSortFinishTime()).ThenReturn(taSortFinishTime
);
Org.Mockito.Mockito.When(attempt.GetState()).ThenReturn(taState);
TaskAttemptReport taReport = Org.Mockito.Mockito.Mock <TaskAttemptReport>();
Org.Mockito.Mockito.When(taReport.GetStartTime()).ThenReturn(taStartTime);
Org.Mockito.Mockito.When(taReport.GetFinishTime()).ThenReturn(taFinishTime);
Org.Mockito.Mockito.When(taReport.GetShuffleFinishTime()).ThenReturn(taShuffleFinishTime
);
Org.Mockito.Mockito.When(taReport.GetSortFinishTime()).ThenReturn(taSortFinishTime
);
Org.Mockito.Mockito.When(taReport.GetContainerId()).ThenReturn(containerId);
Org.Mockito.Mockito.When(taReport.GetProgress()).ThenReturn(1.0f);
Org.Mockito.Mockito.When(taReport.GetStateString()).ThenReturn("Processed 128/128 records <p> \n"
);
Org.Mockito.Mockito.When(taReport.GetTaskAttemptState()).ThenReturn(taState);
Org.Mockito.Mockito.When(taReport.GetDiagnosticInfo()).ThenReturn(string.Empty);
Org.Mockito.Mockito.When(attempt.GetReport()).ThenReturn(taReport);
attempts[taId] = attempt;
Org.Mockito.Mockito.When(task.GetAttempts()).ThenReturn(attempts);
app.SetTask(task);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
Org.Mockito.Mockito.When(job.GetUserName()).ThenReturn("User");
app.SetJob(job);
TestBlocks.AttemptsBlockForTest block = new TestBlocks.AttemptsBlockForTest(this,
app);
block.AddParameter(AMParams.TaskType, "r");
PrintWriter pWriter = new PrintWriter(data);
HtmlBlock.Block html = new BlockForTest(new TestBlocks.HtmlBlockForTest(this), pWriter
, 0, false);
block.Render(html);
pWriter.Flush();
// should be printed information about attempts
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("0 attempt_0_0001_r_000000_0"
));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("SUCCEEDED"));
NUnit.Framework.Assert.IsFalse(data.ToString().Contains("Processed 128/128 records <p> \n"
));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("Processed 128\\/128 records <p> \\n"
));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("_0005_01_000001:attempt_0_0001_r_000000_0:User:"******"100002"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100010"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100011"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100012"));
}
