public virtual void TestCommitPending()
{
MRApp app = new MRApp(1, 0, false, this.GetType().FullName, true);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration());
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task task = it.Next();
app.WaitForState(task, TaskState.Running);
TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next();
app.WaitForState(attempt, TaskAttemptState.Running);
//send the commit pending signal to the task
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(attempt.GetID(), TaskAttemptEventType
.TaCommitPending));
//wait for first attempt to commit pending
app.WaitForState(attempt, TaskAttemptState.CommitPending);
//re-send the commit pending signal to the task
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(attempt.GetID(), TaskAttemptEventType
.TaCommitPending));
//the task attempt should be still at COMMIT_PENDING
app.WaitForState(attempt, TaskAttemptState.CommitPending);
//send the done signal to the task
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task.GetAttempts()
.Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone));
app.WaitForState(job, JobState.Succeeded);
}
public virtual void TestHistoryEvents()
{
Configuration conf = new Configuration();
MRApp app = new TestJobHistoryEvents.MRAppWithHistory(2, 1, true, this.GetType().
FullName, true);
app.Submit(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
JobId jobId = job.GetID();
Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString());
app.WaitForState(job, JobState.Succeeded);
//make sure all events are flushed
app.WaitForState(Service.STATE.Stopped);
/*
* Use HistoryContext to read logged events and verify the number of
* completed maps
*/
HistoryContext context = new JobHistory();
// test start and stop states
((JobHistory)context).Init(conf);
((JobHistory)context).Start();
NUnit.Framework.Assert.IsTrue(context.GetStartTime() > 0);
NUnit.Framework.Assert.AreEqual(((JobHistory)context).GetServiceState(), Service.STATE
.Started);
// get job before stopping JobHistory
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job parsedJob = context.GetJob(jobId);
// stop JobHistory
((JobHistory)context).Stop();
NUnit.Framework.Assert.AreEqual(((JobHistory)context).GetServiceState(), Service.STATE
.Stopped);
NUnit.Framework.Assert.AreEqual("CompletedMaps not correct", 2, parsedJob.GetCompletedMaps
());
NUnit.Framework.Assert.AreEqual(Runtime.GetProperty("user.name"), parsedJob.GetUserName
());
IDictionary <TaskId, Task> tasks = parsedJob.GetTasks();
NUnit.Framework.Assert.AreEqual("No of tasks not correct", 3, tasks.Count);
foreach (Task task in tasks.Values)
{
VerifyTask(task);
}
IDictionary <TaskId, Task> maps = parsedJob.GetTasks(TaskType.Map);
NUnit.Framework.Assert.AreEqual("No of maps not correct", 2, maps.Count);
IDictionary <TaskId, Task> reduces = parsedJob.GetTasks(TaskType.Reduce);
NUnit.Framework.Assert.AreEqual("No of reduces not correct", 1, reduces.Count);
NUnit.Framework.Assert.AreEqual("CompletedReduce not correct", 1, parsedJob.GetCompletedReduces
());
NUnit.Framework.Assert.AreEqual("Job state not currect", JobState.Succeeded, parsedJob
.GetState());
}
public virtual void TestTaskFailWithUnusedContainer()
{
MRApp app = new TestFail.MRAppWithFailingTaskAndUnusedContainer();
Configuration conf = new Configuration();
int maxAttempts = 1;
conf.SetInt(MRJobConfig.MapMaxAttempts, maxAttempts);
// disable uberization (requires entire job to be reattempted, so max for
// subtask attempts is overridden to 1)
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
IDictionary <TaskId, Task> tasks = job.GetTasks();
NUnit.Framework.Assert.AreEqual("Num tasks is not correct", 1, tasks.Count);
Task task = tasks.Values.GetEnumerator().Next();
app.WaitForState(task, TaskState.Scheduled);
IDictionary <TaskAttemptId, TaskAttempt> attempts = tasks.Values.GetEnumerator().Next
().GetAttempts();
NUnit.Framework.Assert.AreEqual("Num attempts is not correct", maxAttempts, attempts
.Count);
TaskAttempt attempt = attempts.Values.GetEnumerator().Next();
app.WaitForInternalState((TaskAttemptImpl)attempt, TaskAttemptStateInternal.Assigned
);
app.GetDispatcher().GetEventHandler().Handle(new TaskAttemptEvent(attempt.GetID()
, TaskAttemptEventType.TaContainerCompleted));
app.WaitForState(job, JobState.Failed);
}
private float GetReduceProgress()
{
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = this._enclosing.myAppContext.GetJob
(this.myAttemptID.GetTaskId().GetJobId());
float runtime = this.GetCodeRuntime();
ICollection <Task> allMapTasks = job.GetTasks(TaskType.Map).Values;
int numberMaps = allMapTasks.Count;
int numberDoneMaps = 0;
foreach (Task mapTask in allMapTasks)
{
if (mapTask.IsFinished())
{
++numberDoneMaps;
}
}
if (numberMaps == numberDoneMaps)
{
this.shuffleCompletedTime = Math.Min(this.shuffleCompletedTime, this._enclosing.clock
.GetTime());
return(Math.Min((float)(this._enclosing.clock.GetTime() - this.shuffleCompletedTime
) / (runtime * 2000.0F) + 0.5F, 1.0F));
}
else
{
return(((float)numberDoneMaps) / numberMaps * 0.5F);
}
}
public virtual void TestKillJob()
{
CountDownLatch latch = new CountDownLatch(1);
MRApp app = new TestKill.BlockingMRApp(1, 0, latch);
//this will start the job but job won't complete as task is
//blocked
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration());
//wait and vailidate for Job to become RUNNING
app.WaitForState(job, JobState.Running);
//send the kill signal to Job
app.GetContext().GetEventHandler().Handle(new JobEvent(job.GetID(), JobEventType.
JobKill));
//unblock Task
latch.CountDown();
//wait and validate for Job to be KILLED
app.WaitForState(job, JobState.Killed);
IDictionary <TaskId, Task> tasks = job.GetTasks();
NUnit.Framework.Assert.AreEqual("No of tasks is not correct", 1, tasks.Count);
Task task = tasks.Values.GetEnumerator().Next();
NUnit.Framework.Assert.AreEqual("Task state not correct", TaskState.Killed, task.
GetReport().GetTaskState());
IDictionary <TaskAttemptId, TaskAttempt> attempts = tasks.Values.GetEnumerator().Next
().GetAttempts();
NUnit.Framework.Assert.AreEqual("No of attempts is not correct", 1, attempts.Count
);
IEnumerator <TaskAttempt> it = attempts.Values.GetEnumerator();
NUnit.Framework.Assert.AreEqual("Attempt state not correct", TaskAttemptState.Killed
, it.Next().GetReport().GetTaskAttemptState());
}
/// <exception cref="System.IO.IOException"/>
public virtual GetTaskReportsResponse GetTaskReports(GetTaskReportsRequest request
)
{
JobId jobId = request.GetJobId();
TaskType taskType = request.GetTaskType();
GetTaskReportsResponse response = this.recordFactory.NewRecordInstance <GetTaskReportsResponse
>();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = this.VerifyAndGetJob(jobId, JobACL
.ViewJob, true);
ICollection <Task> tasks = job.GetTasks(taskType).Values;
MRClientService.Log.Info("Getting task report for " + taskType + " " + jobId +
". Report-size will be " + tasks.Count);
// Take lock to allow only one call, otherwise heap will blow up because
// of counters in the report when there are multiple callers.
lock (this.getTaskReportsLock)
{
foreach (Task task in tasks)
{
response.AddTaskReport(task.GetReport());
}
}
return(response);
}
public virtual void TestHsTasksBlock()
{
Task task = GetTask(0);
IDictionary <TaskId, Task> tasks = new Dictionary <TaskId, Task>();
tasks[task.GetID()] = task;
AppContext ctx = Org.Mockito.Mockito.Mock <AppContext>();
AppForTest app = new AppForTest(ctx);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
Org.Mockito.Mockito.When(job.GetTasks()).ThenReturn(tasks);
app.SetJob(job);
TestBlocks.HsTasksBlockForTest block = new TestBlocks.HsTasksBlockForTest(this, app
);
block.AddParameter(AMParams.TaskType, "r");
PrintWriter pWriter = new PrintWriter(data);
HtmlBlock.Block html = new BlockForTest(new TestBlocks.HtmlBlockForTest(this), pWriter
, 0, false);
block.Render(html);
pWriter.Flush();
// should be printed information about task
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("task_0_0001_r_000000"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("SUCCEEDED"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100001"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100011"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains(string.Empty));
}
public virtual void TestFailTask()
{
//First attempt is failed and second attempt is passed
//The job succeeds.
MRApp app = new TestFail.MockFirstFailingAttemptMRApp(1, 0);
Configuration conf = new Configuration();
// this test requires two task attempts, but uberization overrides max to 1
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Succeeded);
IDictionary <TaskId, Task> tasks = job.GetTasks();
NUnit.Framework.Assert.AreEqual("Num tasks is not correct", 1, tasks.Count);
Task task = tasks.Values.GetEnumerator().Next();
NUnit.Framework.Assert.AreEqual("Task state not correct", TaskState.Succeeded, task
.GetReport().GetTaskState());
IDictionary <TaskAttemptId, TaskAttempt> attempts = tasks.Values.GetEnumerator().Next
().GetAttempts();
NUnit.Framework.Assert.AreEqual("Num attempts is not correct", 2, attempts.Count);
//one attempt must be failed
//and another must have succeeded
IEnumerator <TaskAttempt> it = attempts.Values.GetEnumerator();
NUnit.Framework.Assert.AreEqual("Attempt state not correct", TaskAttemptState.Failed
, it.Next().GetReport().GetTaskAttemptState());
NUnit.Framework.Assert.AreEqual("Attempt state not correct", TaskAttemptState.Succeeded
, it.Next().GetReport().GetTaskAttemptState());
}
public virtual TasksInfo GetJobTasks(HttpServletRequest hsr, string jid, string type
)
{
Init();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = AMWebServices.GetJobFromJobIdString
(jid, ctx);
CheckAccess(job, hsr);
TasksInfo allTasks = new TasksInfo();
foreach (Task task in job.GetTasks().Values)
{
TaskType ttype = null;
if (type != null && !type.IsEmpty())
{
try
{
ttype = MRApps.TaskType(type);
}
catch (YarnRuntimeException)
{
throw new BadRequestException("tasktype must be either m or r");
}
}
if (ttype != null && task.GetType() != ttype)
{
continue;
}
allTasks.Add(new TaskInfo(task));
}
return(allTasks);
}
public virtual void TestTimedOutTask()
{
//All Task attempts are timed out, leading to Job failure
MRApp app = new TestFail.TimeOutTaskMRApp(1, 0);
Configuration conf = new Configuration();
int maxAttempts = 2;
conf.SetInt(MRJobConfig.MapMaxAttempts, maxAttempts);
// disable uberization (requires entire job to be reattempted, so max for
// subtask attempts is overridden to 1)
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Failed);
IDictionary <TaskId, Task> tasks = job.GetTasks();
NUnit.Framework.Assert.AreEqual("Num tasks is not correct", 1, tasks.Count);
Task task = tasks.Values.GetEnumerator().Next();
NUnit.Framework.Assert.AreEqual("Task state not correct", TaskState.Failed, task.
GetReport().GetTaskState());
IDictionary <TaskAttemptId, TaskAttempt> attempts = tasks.Values.GetEnumerator().Next
().GetAttempts();
NUnit.Framework.Assert.AreEqual("Num attempts is not correct", maxAttempts, attempts
.Count);
foreach (TaskAttempt attempt in attempts.Values)
{
NUnit.Framework.Assert.AreEqual("Attempt state not correct", TaskAttemptState.Failed
, attempt.GetReport().GetTaskAttemptState());
}
}
public virtual void TestAMInfosWithoutRecoveryEnabled()
{
int runCount = 0;
MRApp app = new TestRecovery.MRAppWithHistory(1, 0, false, this.GetType().FullName
, true, ++runCount);
Configuration conf = new Configuration();
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
long am1StartTime = app.GetAllAMInfos()[0].GetStartTime();
NUnit.Framework.Assert.AreEqual("No of tasks not correct", 1, job.GetTasks().Count
);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task mapTask = it.Next();
app.WaitForState(mapTask, TaskState.Running);
TaskAttempt taskAttempt = mapTask.GetAttempts().Values.GetEnumerator().Next();
app.WaitForState(taskAttempt, TaskAttemptState.Running);
// stop the app
app.Stop();
// rerun
app = new TestRecovery.MRAppWithHistory(1, 0, false, this.GetType().FullName, false
, ++runCount);
conf = new Configuration();
// in rerun the AMInfo will be recovered from previous run even if recovery
// is not enabled.
conf.SetBoolean(MRJobConfig.MrAmJobRecoveryEnable, false);
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("No of tasks not correct", 1, job.GetTasks().Count
);
it = job.GetTasks().Values.GetEnumerator();
mapTask = it.Next();
// There should be two AMInfos
IList <AMInfo> amInfos = app.GetAllAMInfos();
NUnit.Framework.Assert.AreEqual(2, amInfos.Count);
AMInfo amInfoOne = amInfos[0];
NUnit.Framework.Assert.AreEqual(am1StartTime, amInfoOne.GetStartTime());
app.Stop();
}
//@Test
/// <exception cref="System.Exception"/>
public virtual void TestCompletedMapsForReduceSlowstart()
{
MRApp app = new MRApp(2, 1, false, this.GetType().FullName, true);
Configuration conf = new Configuration();
//after half of the map completion, reduce will start
conf.SetFloat(MRJobConfig.CompletedMapsForReduceSlowstart, 0.5f);
//uberization forces full slowstart (1.0), so disable that
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
//all maps would be running
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 3, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task mapTask1 = it.Next();
Task mapTask2 = it.Next();
Task reduceTask = it.Next();
// all maps must be running
app.WaitForState(mapTask1, TaskState.Running);
app.WaitForState(mapTask2, TaskState.Running);
TaskAttempt task1Attempt = mapTask1.GetAttempts().Values.GetEnumerator().Next();
TaskAttempt task2Attempt = mapTask2.GetAttempts().Values.GetEnumerator().Next();
//before sending the TA_DONE, event make sure attempt has come to
//RUNNING state
app.WaitForState(task1Attempt, TaskAttemptState.Running);
app.WaitForState(task2Attempt, TaskAttemptState.Running);
// reduces must be in NEW state
NUnit.Framework.Assert.AreEqual("Reduce Task state not correct", TaskState.New, reduceTask
.GetReport().GetTaskState());
//send the done signal to the 1st map task
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapTask1.GetAttempts
().Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone));
//wait for first map task to complete
app.WaitForState(mapTask1, TaskState.Succeeded);
//Once the first map completes, it will schedule the reduces
//now reduce must be running
app.WaitForState(reduceTask, TaskState.Running);
//send the done signal to 2nd map and the reduce to complete the job
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapTask2.GetAttempts
().Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone));
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(reduceTask.GetAttempts
().Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone));
app.WaitForState(job, JobState.Succeeded);
}
public virtual void TestJobError()
{
MRApp app = new MRApp(1, 0, false, this.GetType().FullName, true);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration());
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task task = it.Next();
app.WaitForState(task, TaskState.Running);
//send an invalid event on task at current state
app.GetContext().GetEventHandler().Handle(new TaskEvent(task.GetID(), TaskEventType
.TSchedule));
//this must lead to job error
app.WaitForState(job, JobState.Error);
}
private int MaybeScheduleASpeculation(TaskType type)
{
int successes = 0;
long now = clock.GetTime();
ConcurrentMap <JobId, AtomicInteger> containerNeeds = type == TaskType.Map ? mapContainerNeeds
: reduceContainerNeeds;
foreach (KeyValuePair <JobId, AtomicInteger> jobEntry in containerNeeds)
{
// This race conditon is okay. If we skip a speculation attempt we
// should have tried because the event that lowers the number of
// containers needed to zero hasn't come through, it will next time.
// Also, if we miss the fact that the number of containers needed was
// zero but increased due to a failure it's not too bad to launch one
// container prematurely.
if (jobEntry.Value.Get() > 0)
{
continue;
}
int numberSpeculationsAlready = 0;
int numberRunningTasks = 0;
// loop through the tasks of the kind
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = context.GetJob(jobEntry.Key);
IDictionary <TaskId, Task> tasks = job.GetTasks(type);
int numberAllowedSpeculativeTasks = (int)Math.Max(minimumAllowedSpeculativeTasks,
proportionTotalTasksSpeculatable * tasks.Count);
TaskId bestTaskID = null;
long bestSpeculationValue = -1L;
// this loop is potentially pricey.
// TODO track the tasks that are potentially worth looking at
foreach (KeyValuePair <TaskId, Task> taskEntry in tasks)
{
long mySpeculationValue = SpeculationValue(taskEntry.Key, now);
if (mySpeculationValue == AlreadySpeculating)
{
++numberSpeculationsAlready;
}
if (mySpeculationValue != NotRunning)
{
++numberRunningTasks;
}
if (mySpeculationValue > bestSpeculationValue)
{
bestTaskID = taskEntry.Key;
bestSpeculationValue = mySpeculationValue;
}
}
numberAllowedSpeculativeTasks = (int)Math.Max(numberAllowedSpeculativeTasks, proportionRunningTasksSpeculatable
* numberRunningTasks);
// If we found a speculation target, fire it off
if (bestTaskID != null && numberAllowedSpeculativeTasks > numberSpeculationsAlready)
{
AddSpeculativeAttempt(bestTaskID);
++successes;
}
}
return(successes);
}
public virtual void TestKillTaskWaitKillJobAfterTA_DONE()
{
CountDownLatch latch = new CountDownLatch(1);
Dispatcher dispatcher = new TestKill.MyAsyncDispatch(latch, TaskAttemptEventType.
TaDone);
MRApp app = new _MRApp_252(dispatcher, 1, 1, false, this.GetType().FullName, true
);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration());
JobId jobId = app.GetJobId();
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 2, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task mapTask = it.Next();
Task reduceTask = it.Next();
app.WaitForState(mapTask, TaskState.Running);
app.WaitForState(reduceTask, TaskState.Running);
TaskAttempt mapAttempt = mapTask.GetAttempts().Values.GetEnumerator().Next();
app.WaitForState(mapAttempt, TaskAttemptState.Running);
TaskAttempt reduceAttempt = reduceTask.GetAttempts().Values.GetEnumerator().Next(
);
app.WaitForState(reduceAttempt, TaskAttemptState.Running);
// The order in the dispatch event queue, from the oldest to the newest
// TA_DONE
// JOB_KILL
// CONTAINER_REMOTE_CLEANUP ( from TA_DONE's handling )
// T_KILL ( from JOB_KILL's handling )
// TA_CONTAINER_CLEANED ( from CONTAINER_REMOTE_CLEANUP's handling )
// TA_KILL ( from T_KILL's handling )
// T_ATTEMPT_SUCCEEDED ( from TA_CONTAINER_CLEANED's handling )
// T_ATTEMPT_KILLED ( from TA_KILL's handling )
// Finish map
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapAttempt.GetID()
, TaskAttemptEventType.TaDone));
// Now kill the job
app.GetContext().GetEventHandler().Handle(new JobEvent(jobId, JobEventType.JobKill
));
//unblock
latch.CountDown();
app.WaitForInternalState((JobImpl)job, JobStateInternal.Killed);
}
public virtual void TestJobRebootNotLastRetryOnUnregistrationFailure()
{
MRApp app = new MRApp(1, 0, false, this.GetType().FullName, true);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration());
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task task = it.Next();
app.WaitForState(task, TaskState.Running);
//send an reboot event
app.GetContext().GetEventHandler().Handle(new JobEvent(job.GetID(), JobEventType.
JobAmReboot));
// return exteranl state as RUNNING since otherwise the JobClient will
// prematurely exit.
app.WaitForState(job, JobState.Running);
}
public virtual void TestTasksBlock()
{
ApplicationId appId = ApplicationIdPBImpl.NewInstance(0, 1);
JobId jobId = new JobIdPBImpl();
jobId.SetId(0);
jobId.SetAppId(appId);
TaskId taskId = new TaskIdPBImpl();
taskId.SetId(0);
taskId.SetTaskType(TaskType.Map);
taskId.SetJobId(jobId);
Task task = Org.Mockito.Mockito.Mock <Task>();
Org.Mockito.Mockito.When(task.GetID()).ThenReturn(taskId);
TaskReport report = Org.Mockito.Mockito.Mock <TaskReport>();
Org.Mockito.Mockito.When(report.GetProgress()).ThenReturn(0.7f);
Org.Mockito.Mockito.When(report.GetTaskState()).ThenReturn(TaskState.Succeeded);
Org.Mockito.Mockito.When(report.GetStartTime()).ThenReturn(100001L);
Org.Mockito.Mockito.When(report.GetFinishTime()).ThenReturn(100011L);
Org.Mockito.Mockito.When(report.GetStatus()).ThenReturn("Dummy Status \n*");
Org.Mockito.Mockito.When(task.GetReport()).ThenReturn(report);
Org.Mockito.Mockito.When(task.GetType()).ThenReturn(TaskType.Map);
IDictionary <TaskId, Task> tasks = new Dictionary <TaskId, Task>();
tasks[taskId] = task;
AppContext ctx = Org.Mockito.Mockito.Mock <AppContext>();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
Org.Mockito.Mockito.When(job.GetTasks()).ThenReturn(tasks);
Org.Apache.Hadoop.Mapreduce.V2.App.Webapp.App app = new Org.Apache.Hadoop.Mapreduce.V2.App.Webapp.App
(ctx);
app.SetJob(job);
TasksBlockForTest taskBlock = new TasksBlockForTest(app);
taskBlock.AddParameter(AMParams.TaskType, "m");
PrintWriter pWriter = new PrintWriter(data);
HtmlBlock.Block html = new BlockForTest(new TestBlocks.HtmlBlockForTest(this), pWriter
, 0, false);
taskBlock.Render(html);
pWriter.Flush();
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("task_0_0001_m_000000"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("70.00"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("SUCCEEDED"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100001"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100011"));
NUnit.Framework.Assert.IsFalse(data.ToString().Contains("Dummy Status \n*"));
NUnit.Framework.Assert.IsTrue(data.ToString().Contains("Dummy Status \\n*"));
}
public virtual void TestKillTaskAttempt()
{
CountDownLatch latch = new CountDownLatch(1);
MRApp app = new TestKill.BlockingMRApp(2, 0, latch);
//this will start the job but job won't complete as Task is blocked
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration());
//wait and vailidate for Job to become RUNNING
app.WaitForState(job, JobState.Running);
IDictionary <TaskId, Task> tasks = job.GetTasks();
NUnit.Framework.Assert.AreEqual("No of tasks is not correct", 2, tasks.Count);
IEnumerator <Task> it = tasks.Values.GetEnumerator();
Task task1 = it.Next();
Task task2 = it.Next();
//wait for tasks to become running
app.WaitForState(task1, TaskState.Scheduled);
app.WaitForState(task2, TaskState.Scheduled);
//send the kill signal to the first Task's attempt
TaskAttempt attempt = task1.GetAttempts().Values.GetEnumerator().Next();
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(attempt.GetID(), TaskAttemptEventType
.TaKill));
//unblock
latch.CountDown();
//wait and validate for Job to become SUCCEEDED
//job will still succeed
app.WaitForState(job, JobState.Succeeded);
//first Task will have two attempts 1st is killed, 2nd Succeeds
//both Tasks and Job succeeds
NUnit.Framework.Assert.AreEqual("Task state not correct", TaskState.Succeeded, task1
.GetReport().GetTaskState());
NUnit.Framework.Assert.AreEqual("Task state not correct", TaskState.Succeeded, task2
.GetReport().GetTaskState());
IDictionary <TaskAttemptId, TaskAttempt> attempts = task1.GetAttempts();
NUnit.Framework.Assert.AreEqual("No of attempts is not correct", 2, attempts.Count
);
IEnumerator <TaskAttempt> iter = attempts.Values.GetEnumerator();
NUnit.Framework.Assert.AreEqual("Attempt state not correct", TaskAttemptState.Killed
, iter.Next().GetReport().GetTaskAttemptState());
NUnit.Framework.Assert.AreEqual("Attempt state not correct", TaskAttemptState.Succeeded
, iter.Next().GetReport().GetTaskAttemptState());
attempts = task2.GetAttempts();
NUnit.Framework.Assert.AreEqual("No of attempts is not correct", 1, attempts.Count
);
iter = attempts.Values.GetEnumerator();
NUnit.Framework.Assert.AreEqual("Attempt state not correct", TaskAttemptState.Succeeded
, iter.Next().GetReport().GetTaskAttemptState());
}
public virtual void TestKillTaskWait()
{
Dispatcher dispatcher = new _AsyncDispatcher_145();
// Task is asking the reduce TA to kill itself. 'Create' a race
// condition. Make the task succeed and then inform the task that
// TA has succeeded. Once Task gets the TA succeeded event at
// KILL_WAIT, then relay the actual kill signal to TA
// When the TA comes and reports that it is done, send the
// cachedKillEvent
MRApp app = new _MRApp_183(dispatcher, 1, 1, false, this.GetType().FullName, true
);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration());
JobId jobId = app.GetJobId();
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 2, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task mapTask = it.Next();
Task reduceTask = it.Next();
app.WaitForState(mapTask, TaskState.Running);
app.WaitForState(reduceTask, TaskState.Running);
TaskAttempt mapAttempt = mapTask.GetAttempts().Values.GetEnumerator().Next();
app.WaitForState(mapAttempt, TaskAttemptState.Running);
TaskAttempt reduceAttempt = reduceTask.GetAttempts().Values.GetEnumerator().Next(
);
app.WaitForState(reduceAttempt, TaskAttemptState.Running);
// Finish map
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapAttempt.GetID()
, TaskAttemptEventType.TaDone));
app.WaitForState(mapTask, TaskState.Succeeded);
// Now kill the job
app.GetContext().GetEventHandler().Handle(new JobEvent(jobId, JobEventType.JobKill
));
app.WaitForInternalState((JobImpl)job, JobStateInternal.Killed);
}
public virtual void TestJobRebootOnLastRetryOnUnregistrationFailure()
{
// make startCount as 2 since this is last retry which equals to
// DEFAULT_MAX_AM_RETRY
// The last param mocks the unregistration failure
MRApp app = new MRApp(1, 0, false, this.GetType().FullName, true, 2, false);
Configuration conf = new Configuration();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task task = it.Next();
app.WaitForState(task, TaskState.Running);
//send an reboot event
app.GetContext().GetEventHandler().Handle(new JobEvent(job.GetID(), JobEventType.
JobAmReboot));
app.WaitForInternalState((JobImpl)job, JobStateInternal.Reboot);
// return exteranl state as RUNNING if this is the last retry while
// unregistration fails
app.WaitForState(job, JobState.Running);
}
/// <exception cref="System.Exception"/>
public virtual void TestSlowNM()
{
conf = new Configuration();
int maxAttempts = 1;
conf.SetInt(MRJobConfig.MapMaxAttempts, maxAttempts);
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
// set timeout low for the test
conf.SetInt("yarn.rpc.nm-command-timeout", 3000);
conf.Set(YarnConfiguration.IpcRpcImpl, typeof(HadoopYarnProtoRPC).FullName);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
NMTokenSecretManagerInNM tokenSecretManager = new NMTokenSecretManagerInNM();
MasterKey masterKey = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord<MasterKey>();
masterKey.SetBytes(ByteBuffer.Wrap(Sharpen.Runtime.GetBytesForString("key")));
tokenSecretManager.SetMasterKey(masterKey);
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "token");
server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestContainerLauncher.DummyContainerManager
(this), addr, conf, tokenSecretManager, 1);
server.Start();
MRApp app = new TestContainerLauncher.MRAppWithSlowNM(this, tokenSecretManager);
try
{
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
IDictionary<TaskId, Task> tasks = job.GetTasks();
NUnit.Framework.Assert.AreEqual("Num tasks is not correct", 1, tasks.Count);
Task task = tasks.Values.GetEnumerator().Next();
app.WaitForState(task, TaskState.Scheduled);
IDictionary<TaskAttemptId, TaskAttempt> attempts = tasks.Values.GetEnumerator().Next
().GetAttempts();
NUnit.Framework.Assert.AreEqual("Num attempts is not correct", maxAttempts, attempts
.Count);
TaskAttempt attempt = attempts.Values.GetEnumerator().Next();
app.WaitForInternalState((TaskAttemptImpl)attempt, TaskAttemptStateInternal.Assigned
);
app.WaitForState(job, JobState.Failed);
string diagnostics = attempt.GetDiagnostics().ToString();
Log.Info("attempt.getDiagnostics: " + diagnostics);
NUnit.Framework.Assert.IsTrue(diagnostics.Contains("Container launch failed for "
+ "container_0_0000_01_000000 : "));
NUnit.Framework.Assert.IsTrue(diagnostics.Contains("java.net.SocketTimeoutException: 3000 millis timeout while waiting for channel"
));
}
finally
{
server.Stop();
app.Stop();
}
}
public virtual void TestEventsFlushOnStop()
{
Configuration conf = new Configuration();
MRApp app = new TestJobHistoryEvents.MRAppWithSpecialHistoryHandler(1, 0, true, this
.GetType().FullName, true);
app.Submit(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
JobId jobId = job.GetID();
Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString());
app.WaitForState(job, JobState.Succeeded);
// make sure all events are flushed
app.WaitForState(Service.STATE.Stopped);
/*
* Use HistoryContext to read logged events and verify the number of
* completed maps
*/
HistoryContext context = new JobHistory();
((JobHistory)context).Init(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job parsedJob = context.GetJob(jobId);
NUnit.Framework.Assert.AreEqual("CompletedMaps not correct", 1, parsedJob.GetCompletedMaps
());
IDictionary <TaskId, Task> tasks = parsedJob.GetTasks();
NUnit.Framework.Assert.AreEqual("No of tasks not correct", 1, tasks.Count);
VerifyTask(tasks.Values.GetEnumerator().Next());
IDictionary <TaskId, Task> maps = parsedJob.GetTasks(TaskType.Map);
NUnit.Framework.Assert.AreEqual("No of maps not correct", 1, maps.Count);
NUnit.Framework.Assert.AreEqual("Job state not currect", JobState.Succeeded, parsedJob
.GetState());
}
private void GetCounters(AppContext ctx, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
job)
{
if (job == null)
{
return;
}
total = job.GetAllCounters();
bool needTotalCounters = false;
if (total == null)
{
total = new Counters();
needTotalCounters = true;
}
map = new Counters();
reduce = new Counters();
// Get all types of counters
IDictionary <TaskId, Task> tasks = job.GetTasks();
foreach (Task t in tasks.Values)
{
Counters counters = t.GetCounters();
if (counters == null)
{
continue;
}
switch (t.GetType())
{
case TaskType.Map:
{
map.IncrAllCounters(counters);
break;
}
case TaskType.Reduce:
{
reduce.IncrAllCounters(counters);
break;
}
}
if (needTotalCounters)
{
total.IncrAllCounters(counters);
}
}
}
public virtual void TestSingleCounterView()
{
AppContext appContext = new MockAppContext(0, 1, 1, 1);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = appContext.GetAllJobs().Values.GetEnumerator
().Next();
// add a failed task to the job without any counters
Task failedTask = MockJobs.NewTask(job.GetID(), 2, 1, true);
IDictionary <TaskId, Task> tasks = job.GetTasks();
tasks[failedTask.GetID()] = failedTask;
IDictionary <string, string> @params = GetJobParams(appContext);
@params[AMParams.CounterGroup] = "org.apache.hadoop.mapreduce.FileSystemCounter";
@params[AMParams.CounterName] = "HDFS_WRITE_OPS";
WebAppTests.TestPage <AppContext>(typeof(SingleCounterPage), appContext, @params);
}
public virtual void TestContainerPassThrough()
{
MRApp app = new _MRApp_494(this, 0, 1, true, this.GetType().FullName, true);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration());
app.WaitForState(job, JobState.Succeeded);
app.VerifyCompleted();
ICollection <Task> tasks = job.GetTasks().Values;
ICollection <TaskAttempt> taskAttempts = tasks.GetEnumerator().Next().GetAttempts(
).Values;
TaskAttemptImpl taskAttempt = (TaskAttemptImpl)taskAttempts.GetEnumerator().Next(
);
// Container from RM should pass through to the launcher. Container object
// should be the same.
NUnit.Framework.Assert.IsTrue(taskAttempt.container == containerObtainedByContainerLauncher
);
}
/// <exception cref="System.IO.IOException"/>
public virtual GetTaskReportsResponse GetTaskReports(GetTaskReportsRequest request
)
{
JobId jobId = request.GetJobId();
TaskType taskType = request.GetTaskType();
GetTaskReportsResponse response = this.recordFactory.NewRecordInstance <GetTaskReportsResponse
>();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = this.VerifyAndGetJob(jobId, true
);
ICollection <Task> tasks = job.GetTasks(taskType).Values;
foreach (Task task in tasks)
{
response.AddTaskReport(task.GetReport());
}
return(response);
}
public virtual void TestAverageReduceTime()
{
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <CompletedJob
>();
Task task1 = Org.Mockito.Mockito.Mock <Task>();
Task task2 = Org.Mockito.Mockito.Mock <Task>();
JobId jobId = MRBuilderUtils.NewJobId(1L, 1, 1);
TaskId taskId1 = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Reduce);
TaskId taskId2 = MRBuilderUtils.NewTaskId(jobId, 2, TaskType.Reduce);
TaskAttemptId taskAttemptId1 = MRBuilderUtils.NewTaskAttemptId(taskId1, 1);
TaskAttemptId taskAttemptId2 = MRBuilderUtils.NewTaskAttemptId(taskId2, 2);
TaskAttempt taskAttempt1 = Org.Mockito.Mockito.Mock <TaskAttempt>();
TaskAttempt taskAttempt2 = Org.Mockito.Mockito.Mock <TaskAttempt>();
JobReport jobReport = Org.Mockito.Mockito.Mock <JobReport>();
Org.Mockito.Mockito.When(taskAttempt1.GetState()).ThenReturn(TaskAttemptState.Succeeded
);
Org.Mockito.Mockito.When(taskAttempt1.GetLaunchTime()).ThenReturn(0L);
Org.Mockito.Mockito.When(taskAttempt1.GetShuffleFinishTime()).ThenReturn(4L);
Org.Mockito.Mockito.When(taskAttempt1.GetSortFinishTime()).ThenReturn(6L);
Org.Mockito.Mockito.When(taskAttempt1.GetFinishTime()).ThenReturn(8L);
Org.Mockito.Mockito.When(taskAttempt2.GetState()).ThenReturn(TaskAttemptState.Succeeded
);
Org.Mockito.Mockito.When(taskAttempt2.GetLaunchTime()).ThenReturn(5L);
Org.Mockito.Mockito.When(taskAttempt2.GetShuffleFinishTime()).ThenReturn(10L);
Org.Mockito.Mockito.When(taskAttempt2.GetSortFinishTime()).ThenReturn(22L);
Org.Mockito.Mockito.When(taskAttempt2.GetFinishTime()).ThenReturn(42L);
Org.Mockito.Mockito.When(task1.GetType()).ThenReturn(TaskType.Reduce);
Org.Mockito.Mockito.When(task2.GetType()).ThenReturn(TaskType.Reduce);
Org.Mockito.Mockito.When(task1.GetAttempts()).ThenReturn(new _Dictionary_120(taskAttemptId1
, taskAttempt1));
Org.Mockito.Mockito.When(task2.GetAttempts()).ThenReturn(new _Dictionary_123(taskAttemptId2
, taskAttempt2));
Org.Mockito.Mockito.When(job.GetTasks()).ThenReturn(new _Dictionary_127(taskId1,
task1, taskId2, task2));
Org.Mockito.Mockito.When(job.GetID()).ThenReturn(jobId);
Org.Mockito.Mockito.When(job.GetReport()).ThenReturn(jobReport);
Org.Mockito.Mockito.When(job.GetName()).ThenReturn("TestJobInfo");
Org.Mockito.Mockito.When(job.GetState()).ThenReturn(JobState.Succeeded);
JobInfo jobInfo = new JobInfo(job);
NUnit.Framework.Assert.AreEqual(11L, jobInfo.GetAvgReduceTime());
}
public virtual void Setup()
{
this.conf = new JobConf();
this.conf.Set(CommonConfigurationKeys.HadoopSecurityGroupMapping, typeof(TestHsWebServicesAcls.NullGroupsProvider
).FullName);
this.conf.SetBoolean(MRConfig.MrAclsEnabled, true);
Groups.GetUserToGroupsMappingService(conf);
this.ctx = BuildHistoryContext(this.conf);
WebApp webApp = Org.Mockito.Mockito.Mock <HsWebApp>();
Org.Mockito.Mockito.When(webApp.Name()).ThenReturn("hsmockwebapp");
this.hsWebServices = new HsWebServices(ctx, conf, webApp);
this.hsWebServices.SetResponse(Org.Mockito.Mockito.Mock <HttpServletResponse>());
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = ctx.GetAllJobs().Values.GetEnumerator
().Next();
this.jobIdStr = job.GetID().ToString();
Task task = job.GetTasks().Values.GetEnumerator().Next();
this.taskIdStr = task.GetID().ToString();
this.taskAttemptIdStr = task.GetAttempts().Keys.GetEnumerator().Next().ToString();
}
public virtual void VerifyAMTaskXML(NodeList nodes, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
job)
{
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 2, nodes.GetLength
());
foreach (Task task in job.GetTasks().Values)
{
TaskId id = task.GetID();
string tid = MRApps.ToString(id);
bool found = false;
for (int i = 0; i < nodes.GetLength(); i++)
{
Element element = (Element)nodes.Item(i);
if (tid.Matches(WebServicesTestUtils.GetXmlString(element, "id")))
{
found = true;
VerifyAMSingleTaskXML(element, task);
}
}
NUnit.Framework.Assert.IsTrue("task with id: " + tid + " not in web service output"
, found);
}
}
/*
* (non-Javadoc)
* @see org.apache.hadoop.mapreduce.v2.hs.webapp.HsTaskPage.AttemptsBlock#getTaskAttempts()
* @return the attempts that are for a given job and a specific type/state.
*/
protected internal override ICollection <TaskAttempt> GetTaskAttempts()
{
IList <TaskAttempt> fewTaskAttemps = new AList <TaskAttempt>();
string taskTypeStr = $(AMParams.TaskType);
TaskType taskType = MRApps.TaskType(taskTypeStr);
string attemptStateStr = $(AMParams.AttemptState);
MRApps.TaskAttemptStateUI neededState = MRApps.TaskAttemptState(attemptStateStr);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job j = app.GetJob();
IDictionary <TaskId, Task> tasks = j.GetTasks(taskType);
foreach (Task task in tasks.Values)
{
IDictionary <TaskAttemptId, TaskAttempt> attempts = task.GetAttempts();
foreach (TaskAttempt attempt in attempts.Values)
{
if (neededState.CorrespondsTo(attempt.GetState()))
{
fewTaskAttemps.AddItem(attempt);
}
}
}
return(fewTaskAttemps);
}
