public virtual void TestCommitPending() { MRApp app = new MRApp(1, 0, false, this.GetType().FullName, true); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration()); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task task = it.Next(); app.WaitForState(task, TaskState.Running); TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next(); app.WaitForState(attempt, TaskAttemptState.Running); //send the commit pending signal to the task app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(attempt.GetID(), TaskAttemptEventType .TaCommitPending)); //wait for first attempt to commit pending app.WaitForState(attempt, TaskAttemptState.CommitPending); //re-send the commit pending signal to the task app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(attempt.GetID(), TaskAttemptEventType .TaCommitPending)); //the task attempt should be still at COMMIT_PENDING app.WaitForState(attempt, TaskAttemptState.CommitPending); //send the done signal to the task app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task.GetAttempts() .Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone)); app.WaitForState(job, JobState.Succeeded); }
public virtual void TestHistoryEvents() { Configuration conf = new Configuration(); MRApp app = new TestJobHistoryEvents.MRAppWithHistory(2, 1, true, this.GetType(). FullName, true); app.Submit(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); JobId jobId = job.GetID(); Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString()); app.WaitForState(job, JobState.Succeeded); //make sure all events are flushed app.WaitForState(Service.STATE.Stopped); /* * Use HistoryContext to read logged events and verify the number of * completed maps */ HistoryContext context = new JobHistory(); // test start and stop states ((JobHistory)context).Init(conf); ((JobHistory)context).Start(); NUnit.Framework.Assert.IsTrue(context.GetStartTime() > 0); NUnit.Framework.Assert.AreEqual(((JobHistory)context).GetServiceState(), Service.STATE .Started); // get job before stopping JobHistory Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job parsedJob = context.GetJob(jobId); // stop JobHistory ((JobHistory)context).Stop(); NUnit.Framework.Assert.AreEqual(((JobHistory)context).GetServiceState(), Service.STATE .Stopped); NUnit.Framework.Assert.AreEqual("CompletedMaps not correct", 2, parsedJob.GetCompletedMaps ()); NUnit.Framework.Assert.AreEqual(Runtime.GetProperty("user.name"), parsedJob.GetUserName ()); IDictionary <TaskId, Task> tasks = parsedJob.GetTasks(); NUnit.Framework.Assert.AreEqual("No of tasks not correct", 3, tasks.Count); foreach (Task task in tasks.Values) { VerifyTask(task); } IDictionary <TaskId, Task> maps = parsedJob.GetTasks(TaskType.Map); NUnit.Framework.Assert.AreEqual("No of maps not correct", 2, maps.Count); IDictionary <TaskId, Task> reduces = parsedJob.GetTasks(TaskType.Reduce); NUnit.Framework.Assert.AreEqual("No of reduces not correct", 1, reduces.Count); NUnit.Framework.Assert.AreEqual("CompletedReduce not correct", 1, parsedJob.GetCompletedReduces ()); NUnit.Framework.Assert.AreEqual("Job state not currect", JobState.Succeeded, parsedJob .GetState()); }
public virtual void TestTaskFailWithUnusedContainer() { MRApp app = new TestFail.MRAppWithFailingTaskAndUnusedContainer(); Configuration conf = new Configuration(); int maxAttempts = 1; conf.SetInt(MRJobConfig.MapMaxAttempts, maxAttempts); // disable uberization (requires entire job to be reattempted, so max for // subtask attempts is overridden to 1) conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Running); IDictionary <TaskId, Task> tasks = job.GetTasks(); NUnit.Framework.Assert.AreEqual("Num tasks is not correct", 1, tasks.Count); Task task = tasks.Values.GetEnumerator().Next(); app.WaitForState(task, TaskState.Scheduled); IDictionary <TaskAttemptId, TaskAttempt> attempts = tasks.Values.GetEnumerator().Next ().GetAttempts(); NUnit.Framework.Assert.AreEqual("Num attempts is not correct", maxAttempts, attempts .Count); TaskAttempt attempt = attempts.Values.GetEnumerator().Next(); app.WaitForInternalState((TaskAttemptImpl)attempt, TaskAttemptStateInternal.Assigned ); app.GetDispatcher().GetEventHandler().Handle(new TaskAttemptEvent(attempt.GetID() , TaskAttemptEventType.TaContainerCompleted)); app.WaitForState(job, JobState.Failed); }
private float GetReduceProgress() { Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = this._enclosing.myAppContext.GetJob (this.myAttemptID.GetTaskId().GetJobId()); float runtime = this.GetCodeRuntime(); ICollection <Task> allMapTasks = job.GetTasks(TaskType.Map).Values; int numberMaps = allMapTasks.Count; int numberDoneMaps = 0; foreach (Task mapTask in allMapTasks) { if (mapTask.IsFinished()) { ++numberDoneMaps; } } if (numberMaps == numberDoneMaps) { this.shuffleCompletedTime = Math.Min(this.shuffleCompletedTime, this._enclosing.clock .GetTime()); return(Math.Min((float)(this._enclosing.clock.GetTime() - this.shuffleCompletedTime ) / (runtime * 2000.0F) + 0.5F, 1.0F)); } else { return(((float)numberDoneMaps) / numberMaps * 0.5F); } }
public virtual void TestKillJob() { CountDownLatch latch = new CountDownLatch(1); MRApp app = new TestKill.BlockingMRApp(1, 0, latch); //this will start the job but job won't complete as task is //blocked Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration()); //wait and vailidate for Job to become RUNNING app.WaitForState(job, JobState.Running); //send the kill signal to Job app.GetContext().GetEventHandler().Handle(new JobEvent(job.GetID(), JobEventType. JobKill)); //unblock Task latch.CountDown(); //wait and validate for Job to be KILLED app.WaitForState(job, JobState.Killed); IDictionary <TaskId, Task> tasks = job.GetTasks(); NUnit.Framework.Assert.AreEqual("No of tasks is not correct", 1, tasks.Count); Task task = tasks.Values.GetEnumerator().Next(); NUnit.Framework.Assert.AreEqual("Task state not correct", TaskState.Killed, task. GetReport().GetTaskState()); IDictionary <TaskAttemptId, TaskAttempt> attempts = tasks.Values.GetEnumerator().Next ().GetAttempts(); NUnit.Framework.Assert.AreEqual("No of attempts is not correct", 1, attempts.Count ); IEnumerator <TaskAttempt> it = attempts.Values.GetEnumerator(); NUnit.Framework.Assert.AreEqual("Attempt state not correct", TaskAttemptState.Killed , it.Next().GetReport().GetTaskAttemptState()); }
/// <exception cref="System.IO.IOException"/> public virtual GetTaskReportsResponse GetTaskReports(GetTaskReportsRequest request ) { JobId jobId = request.GetJobId(); TaskType taskType = request.GetTaskType(); GetTaskReportsResponse response = this.recordFactory.NewRecordInstance <GetTaskReportsResponse >(); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = this.VerifyAndGetJob(jobId, JobACL .ViewJob, true); ICollection <Task> tasks = job.GetTasks(taskType).Values; MRClientService.Log.Info("Getting task report for " + taskType + " " + jobId + ". Report-size will be " + tasks.Count); // Take lock to allow only one call, otherwise heap will blow up because // of counters in the report when there are multiple callers. lock (this.getTaskReportsLock) { foreach (Task task in tasks) { response.AddTaskReport(task.GetReport()); } } return(response); }
public virtual void TestHsTasksBlock() { Task task = GetTask(0); IDictionary <TaskId, Task> tasks = new Dictionary <TaskId, Task>(); tasks[task.GetID()] = task; AppContext ctx = Org.Mockito.Mockito.Mock <AppContext>(); AppForTest app = new AppForTest(ctx); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job >(); Org.Mockito.Mockito.When(job.GetTasks()).ThenReturn(tasks); app.SetJob(job); TestBlocks.HsTasksBlockForTest block = new TestBlocks.HsTasksBlockForTest(this, app ); block.AddParameter(AMParams.TaskType, "r"); PrintWriter pWriter = new PrintWriter(data); HtmlBlock.Block html = new BlockForTest(new TestBlocks.HtmlBlockForTest(this), pWriter , 0, false); block.Render(html); pWriter.Flush(); // should be printed information about task NUnit.Framework.Assert.IsTrue(data.ToString().Contains("task_0_0001_r_000000")); NUnit.Framework.Assert.IsTrue(data.ToString().Contains("SUCCEEDED")); NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100001")); NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100011")); NUnit.Framework.Assert.IsTrue(data.ToString().Contains(string.Empty)); }
public virtual void TestFailTask() { //First attempt is failed and second attempt is passed //The job succeeds. MRApp app = new TestFail.MockFirstFailingAttemptMRApp(1, 0); Configuration conf = new Configuration(); // this test requires two task attempts, but uberization overrides max to 1 conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Succeeded); IDictionary <TaskId, Task> tasks = job.GetTasks(); NUnit.Framework.Assert.AreEqual("Num tasks is not correct", 1, tasks.Count); Task task = tasks.Values.GetEnumerator().Next(); NUnit.Framework.Assert.AreEqual("Task state not correct", TaskState.Succeeded, task .GetReport().GetTaskState()); IDictionary <TaskAttemptId, TaskAttempt> attempts = tasks.Values.GetEnumerator().Next ().GetAttempts(); NUnit.Framework.Assert.AreEqual("Num attempts is not correct", 2, attempts.Count); //one attempt must be failed //and another must have succeeded IEnumerator <TaskAttempt> it = attempts.Values.GetEnumerator(); NUnit.Framework.Assert.AreEqual("Attempt state not correct", TaskAttemptState.Failed , it.Next().GetReport().GetTaskAttemptState()); NUnit.Framework.Assert.AreEqual("Attempt state not correct", TaskAttemptState.Succeeded , it.Next().GetReport().GetTaskAttemptState()); }
public virtual TasksInfo GetJobTasks(HttpServletRequest hsr, string jid, string type ) { Init(); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = AMWebServices.GetJobFromJobIdString (jid, ctx); CheckAccess(job, hsr); TasksInfo allTasks = new TasksInfo(); foreach (Task task in job.GetTasks().Values) { TaskType ttype = null; if (type != null && !type.IsEmpty()) { try { ttype = MRApps.TaskType(type); } catch (YarnRuntimeException) { throw new BadRequestException("tasktype must be either m or r"); } } if (ttype != null && task.GetType() != ttype) { continue; } allTasks.Add(new TaskInfo(task)); } return(allTasks); }
public virtual void TestTimedOutTask() { //All Task attempts are timed out, leading to Job failure MRApp app = new TestFail.TimeOutTaskMRApp(1, 0); Configuration conf = new Configuration(); int maxAttempts = 2; conf.SetInt(MRJobConfig.MapMaxAttempts, maxAttempts); // disable uberization (requires entire job to be reattempted, so max for // subtask attempts is overridden to 1) conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Failed); IDictionary <TaskId, Task> tasks = job.GetTasks(); NUnit.Framework.Assert.AreEqual("Num tasks is not correct", 1, tasks.Count); Task task = tasks.Values.GetEnumerator().Next(); NUnit.Framework.Assert.AreEqual("Task state not correct", TaskState.Failed, task. GetReport().GetTaskState()); IDictionary <TaskAttemptId, TaskAttempt> attempts = tasks.Values.GetEnumerator().Next ().GetAttempts(); NUnit.Framework.Assert.AreEqual("Num attempts is not correct", maxAttempts, attempts .Count); foreach (TaskAttempt attempt in attempts.Values) { NUnit.Framework.Assert.AreEqual("Attempt state not correct", TaskAttemptState.Failed , attempt.GetReport().GetTaskAttemptState()); } }
public virtual void TestAMInfosWithoutRecoveryEnabled() { int runCount = 0; MRApp app = new TestRecovery.MRAppWithHistory(1, 0, false, this.GetType().FullName , true, ++runCount); Configuration conf = new Configuration(); conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Running); long am1StartTime = app.GetAllAMInfos()[0].GetStartTime(); NUnit.Framework.Assert.AreEqual("No of tasks not correct", 1, job.GetTasks().Count ); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task mapTask = it.Next(); app.WaitForState(mapTask, TaskState.Running); TaskAttempt taskAttempt = mapTask.GetAttempts().Values.GetEnumerator().Next(); app.WaitForState(taskAttempt, TaskAttemptState.Running); // stop the app app.Stop(); // rerun app = new TestRecovery.MRAppWithHistory(1, 0, false, this.GetType().FullName, false , ++runCount); conf = new Configuration(); // in rerun the AMInfo will be recovered from previous run even if recovery // is not enabled. conf.SetBoolean(MRJobConfig.MrAmJobRecoveryEnable, false); conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false); job = app.Submit(conf); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("No of tasks not correct", 1, job.GetTasks().Count ); it = job.GetTasks().Values.GetEnumerator(); mapTask = it.Next(); // There should be two AMInfos IList <AMInfo> amInfos = app.GetAllAMInfos(); NUnit.Framework.Assert.AreEqual(2, amInfos.Count); AMInfo amInfoOne = amInfos[0]; NUnit.Framework.Assert.AreEqual(am1StartTime, amInfoOne.GetStartTime()); app.Stop(); }
//@Test /// <exception cref="System.Exception"/> public virtual void TestCompletedMapsForReduceSlowstart() { MRApp app = new MRApp(2, 1, false, this.GetType().FullName, true); Configuration conf = new Configuration(); //after half of the map completion, reduce will start conf.SetFloat(MRJobConfig.CompletedMapsForReduceSlowstart, 0.5f); //uberization forces full slowstart (1.0), so disable that conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Running); //all maps would be running NUnit.Framework.Assert.AreEqual("Num tasks not correct", 3, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task mapTask1 = it.Next(); Task mapTask2 = it.Next(); Task reduceTask = it.Next(); // all maps must be running app.WaitForState(mapTask1, TaskState.Running); app.WaitForState(mapTask2, TaskState.Running); TaskAttempt task1Attempt = mapTask1.GetAttempts().Values.GetEnumerator().Next(); TaskAttempt task2Attempt = mapTask2.GetAttempts().Values.GetEnumerator().Next(); //before sending the TA_DONE, event make sure attempt has come to //RUNNING state app.WaitForState(task1Attempt, TaskAttemptState.Running); app.WaitForState(task2Attempt, TaskAttemptState.Running); // reduces must be in NEW state NUnit.Framework.Assert.AreEqual("Reduce Task state not correct", TaskState.New, reduceTask .GetReport().GetTaskState()); //send the done signal to the 1st map task app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapTask1.GetAttempts ().Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone)); //wait for first map task to complete app.WaitForState(mapTask1, TaskState.Succeeded); //Once the first map completes, it will schedule the reduces //now reduce must be running app.WaitForState(reduceTask, TaskState.Running); //send the done signal to 2nd map and the reduce to complete the job app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapTask2.GetAttempts ().Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone)); app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(reduceTask.GetAttempts ().Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone)); app.WaitForState(job, JobState.Succeeded); }
public virtual void TestJobError() { MRApp app = new MRApp(1, 0, false, this.GetType().FullName, true); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration()); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task task = it.Next(); app.WaitForState(task, TaskState.Running); //send an invalid event on task at current state app.GetContext().GetEventHandler().Handle(new TaskEvent(task.GetID(), TaskEventType .TSchedule)); //this must lead to job error app.WaitForState(job, JobState.Error); }
private int MaybeScheduleASpeculation(TaskType type) { int successes = 0; long now = clock.GetTime(); ConcurrentMap <JobId, AtomicInteger> containerNeeds = type == TaskType.Map ? mapContainerNeeds : reduceContainerNeeds; foreach (KeyValuePair <JobId, AtomicInteger> jobEntry in containerNeeds) { // This race conditon is okay. If we skip a speculation attempt we // should have tried because the event that lowers the number of // containers needed to zero hasn't come through, it will next time. // Also, if we miss the fact that the number of containers needed was // zero but increased due to a failure it's not too bad to launch one // container prematurely. if (jobEntry.Value.Get() > 0) { continue; } int numberSpeculationsAlready = 0; int numberRunningTasks = 0; // loop through the tasks of the kind Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = context.GetJob(jobEntry.Key); IDictionary <TaskId, Task> tasks = job.GetTasks(type); int numberAllowedSpeculativeTasks = (int)Math.Max(minimumAllowedSpeculativeTasks, proportionTotalTasksSpeculatable * tasks.Count); TaskId bestTaskID = null; long bestSpeculationValue = -1L; // this loop is potentially pricey. // TODO track the tasks that are potentially worth looking at foreach (KeyValuePair <TaskId, Task> taskEntry in tasks) { long mySpeculationValue = SpeculationValue(taskEntry.Key, now); if (mySpeculationValue == AlreadySpeculating) { ++numberSpeculationsAlready; } if (mySpeculationValue != NotRunning) { ++numberRunningTasks; } if (mySpeculationValue > bestSpeculationValue) { bestTaskID = taskEntry.Key; bestSpeculationValue = mySpeculationValue; } } numberAllowedSpeculativeTasks = (int)Math.Max(numberAllowedSpeculativeTasks, proportionRunningTasksSpeculatable * numberRunningTasks); // If we found a speculation target, fire it off if (bestTaskID != null && numberAllowedSpeculativeTasks > numberSpeculationsAlready) { AddSpeculativeAttempt(bestTaskID); ++successes; } } return(successes); }
public virtual void TestKillTaskWaitKillJobAfterTA_DONE() { CountDownLatch latch = new CountDownLatch(1); Dispatcher dispatcher = new TestKill.MyAsyncDispatch(latch, TaskAttemptEventType. TaDone); MRApp app = new _MRApp_252(dispatcher, 1, 1, false, this.GetType().FullName, true ); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration()); JobId jobId = app.GetJobId(); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("Num tasks not correct", 2, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task mapTask = it.Next(); Task reduceTask = it.Next(); app.WaitForState(mapTask, TaskState.Running); app.WaitForState(reduceTask, TaskState.Running); TaskAttempt mapAttempt = mapTask.GetAttempts().Values.GetEnumerator().Next(); app.WaitForState(mapAttempt, TaskAttemptState.Running); TaskAttempt reduceAttempt = reduceTask.GetAttempts().Values.GetEnumerator().Next( ); app.WaitForState(reduceAttempt, TaskAttemptState.Running); // The order in the dispatch event queue, from the oldest to the newest // TA_DONE // JOB_KILL // CONTAINER_REMOTE_CLEANUP ( from TA_DONE's handling ) // T_KILL ( from JOB_KILL's handling ) // TA_CONTAINER_CLEANED ( from CONTAINER_REMOTE_CLEANUP's handling ) // TA_KILL ( from T_KILL's handling ) // T_ATTEMPT_SUCCEEDED ( from TA_CONTAINER_CLEANED's handling ) // T_ATTEMPT_KILLED ( from TA_KILL's handling ) // Finish map app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapAttempt.GetID() , TaskAttemptEventType.TaDone)); // Now kill the job app.GetContext().GetEventHandler().Handle(new JobEvent(jobId, JobEventType.JobKill )); //unblock latch.CountDown(); app.WaitForInternalState((JobImpl)job, JobStateInternal.Killed); }
public virtual void TestJobRebootNotLastRetryOnUnregistrationFailure() { MRApp app = new MRApp(1, 0, false, this.GetType().FullName, true); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration()); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task task = it.Next(); app.WaitForState(task, TaskState.Running); //send an reboot event app.GetContext().GetEventHandler().Handle(new JobEvent(job.GetID(), JobEventType. JobAmReboot)); // return exteranl state as RUNNING since otherwise the JobClient will // prematurely exit. app.WaitForState(job, JobState.Running); }
public virtual void TestTasksBlock() { ApplicationId appId = ApplicationIdPBImpl.NewInstance(0, 1); JobId jobId = new JobIdPBImpl(); jobId.SetId(0); jobId.SetAppId(appId); TaskId taskId = new TaskIdPBImpl(); taskId.SetId(0); taskId.SetTaskType(TaskType.Map); taskId.SetJobId(jobId); Task task = Org.Mockito.Mockito.Mock <Task>(); Org.Mockito.Mockito.When(task.GetID()).ThenReturn(taskId); TaskReport report = Org.Mockito.Mockito.Mock <TaskReport>(); Org.Mockito.Mockito.When(report.GetProgress()).ThenReturn(0.7f); Org.Mockito.Mockito.When(report.GetTaskState()).ThenReturn(TaskState.Succeeded); Org.Mockito.Mockito.When(report.GetStartTime()).ThenReturn(100001L); Org.Mockito.Mockito.When(report.GetFinishTime()).ThenReturn(100011L); Org.Mockito.Mockito.When(report.GetStatus()).ThenReturn("Dummy Status \n*"); Org.Mockito.Mockito.When(task.GetReport()).ThenReturn(report); Org.Mockito.Mockito.When(task.GetType()).ThenReturn(TaskType.Map); IDictionary <TaskId, Task> tasks = new Dictionary <TaskId, Task>(); tasks[taskId] = task; AppContext ctx = Org.Mockito.Mockito.Mock <AppContext>(); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job >(); Org.Mockito.Mockito.When(job.GetTasks()).ThenReturn(tasks); Org.Apache.Hadoop.Mapreduce.V2.App.Webapp.App app = new Org.Apache.Hadoop.Mapreduce.V2.App.Webapp.App (ctx); app.SetJob(job); TasksBlockForTest taskBlock = new TasksBlockForTest(app); taskBlock.AddParameter(AMParams.TaskType, "m"); PrintWriter pWriter = new PrintWriter(data); HtmlBlock.Block html = new BlockForTest(new TestBlocks.HtmlBlockForTest(this), pWriter , 0, false); taskBlock.Render(html); pWriter.Flush(); NUnit.Framework.Assert.IsTrue(data.ToString().Contains("task_0_0001_m_000000")); NUnit.Framework.Assert.IsTrue(data.ToString().Contains("70.00")); NUnit.Framework.Assert.IsTrue(data.ToString().Contains("SUCCEEDED")); NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100001")); NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100011")); NUnit.Framework.Assert.IsFalse(data.ToString().Contains("Dummy Status \n*")); NUnit.Framework.Assert.IsTrue(data.ToString().Contains("Dummy Status \\n*")); }
public virtual void TestKillTaskAttempt() { CountDownLatch latch = new CountDownLatch(1); MRApp app = new TestKill.BlockingMRApp(2, 0, latch); //this will start the job but job won't complete as Task is blocked Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration()); //wait and vailidate for Job to become RUNNING app.WaitForState(job, JobState.Running); IDictionary <TaskId, Task> tasks = job.GetTasks(); NUnit.Framework.Assert.AreEqual("No of tasks is not correct", 2, tasks.Count); IEnumerator <Task> it = tasks.Values.GetEnumerator(); Task task1 = it.Next(); Task task2 = it.Next(); //wait for tasks to become running app.WaitForState(task1, TaskState.Scheduled); app.WaitForState(task2, TaskState.Scheduled); //send the kill signal to the first Task's attempt TaskAttempt attempt = task1.GetAttempts().Values.GetEnumerator().Next(); app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(attempt.GetID(), TaskAttemptEventType .TaKill)); //unblock latch.CountDown(); //wait and validate for Job to become SUCCEEDED //job will still succeed app.WaitForState(job, JobState.Succeeded); //first Task will have two attempts 1st is killed, 2nd Succeeds //both Tasks and Job succeeds NUnit.Framework.Assert.AreEqual("Task state not correct", TaskState.Succeeded, task1 .GetReport().GetTaskState()); NUnit.Framework.Assert.AreEqual("Task state not correct", TaskState.Succeeded, task2 .GetReport().GetTaskState()); IDictionary <TaskAttemptId, TaskAttempt> attempts = task1.GetAttempts(); NUnit.Framework.Assert.AreEqual("No of attempts is not correct", 2, attempts.Count ); IEnumerator <TaskAttempt> iter = attempts.Values.GetEnumerator(); NUnit.Framework.Assert.AreEqual("Attempt state not correct", TaskAttemptState.Killed , iter.Next().GetReport().GetTaskAttemptState()); NUnit.Framework.Assert.AreEqual("Attempt state not correct", TaskAttemptState.Succeeded , iter.Next().GetReport().GetTaskAttemptState()); attempts = task2.GetAttempts(); NUnit.Framework.Assert.AreEqual("No of attempts is not correct", 1, attempts.Count ); iter = attempts.Values.GetEnumerator(); NUnit.Framework.Assert.AreEqual("Attempt state not correct", TaskAttemptState.Succeeded , iter.Next().GetReport().GetTaskAttemptState()); }
public virtual void TestKillTaskWait() { Dispatcher dispatcher = new _AsyncDispatcher_145(); // Task is asking the reduce TA to kill itself. 'Create' a race // condition. Make the task succeed and then inform the task that // TA has succeeded. Once Task gets the TA succeeded event at // KILL_WAIT, then relay the actual kill signal to TA // When the TA comes and reports that it is done, send the // cachedKillEvent MRApp app = new _MRApp_183(dispatcher, 1, 1, false, this.GetType().FullName, true ); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration()); JobId jobId = app.GetJobId(); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("Num tasks not correct", 2, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task mapTask = it.Next(); Task reduceTask = it.Next(); app.WaitForState(mapTask, TaskState.Running); app.WaitForState(reduceTask, TaskState.Running); TaskAttempt mapAttempt = mapTask.GetAttempts().Values.GetEnumerator().Next(); app.WaitForState(mapAttempt, TaskAttemptState.Running); TaskAttempt reduceAttempt = reduceTask.GetAttempts().Values.GetEnumerator().Next( ); app.WaitForState(reduceAttempt, TaskAttemptState.Running); // Finish map app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapAttempt.GetID() , TaskAttemptEventType.TaDone)); app.WaitForState(mapTask, TaskState.Succeeded); // Now kill the job app.GetContext().GetEventHandler().Handle(new JobEvent(jobId, JobEventType.JobKill )); app.WaitForInternalState((JobImpl)job, JobStateInternal.Killed); }
public virtual void TestJobRebootOnLastRetryOnUnregistrationFailure() { // make startCount as 2 since this is last retry which equals to // DEFAULT_MAX_AM_RETRY // The last param mocks the unregistration failure MRApp app = new MRApp(1, 0, false, this.GetType().FullName, true, 2, false); Configuration conf = new Configuration(); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task task = it.Next(); app.WaitForState(task, TaskState.Running); //send an reboot event app.GetContext().GetEventHandler().Handle(new JobEvent(job.GetID(), JobEventType. JobAmReboot)); app.WaitForInternalState((JobImpl)job, JobStateInternal.Reboot); // return exteranl state as RUNNING if this is the last retry while // unregistration fails app.WaitForState(job, JobState.Running); }
/// <exception cref="System.Exception"/> public virtual void TestSlowNM() { conf = new Configuration(); int maxAttempts = 1; conf.SetInt(MRJobConfig.MapMaxAttempts, maxAttempts); conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false); // set timeout low for the test conf.SetInt("yarn.rpc.nm-command-timeout", 3000); conf.Set(YarnConfiguration.IpcRpcImpl, typeof(HadoopYarnProtoRPC).FullName); YarnRPC rpc = YarnRPC.Create(conf); string bindAddr = "localhost:0"; IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr); NMTokenSecretManagerInNM tokenSecretManager = new NMTokenSecretManagerInNM(); MasterKey masterKey = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord<MasterKey>(); masterKey.SetBytes(ByteBuffer.Wrap(Sharpen.Runtime.GetBytesForString("key"))); tokenSecretManager.SetMasterKey(masterKey); conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "token"); server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestContainerLauncher.DummyContainerManager (this), addr, conf, tokenSecretManager, 1); server.Start(); MRApp app = new TestContainerLauncher.MRAppWithSlowNM(this, tokenSecretManager); try { Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Running); IDictionary<TaskId, Task> tasks = job.GetTasks(); NUnit.Framework.Assert.AreEqual("Num tasks is not correct", 1, tasks.Count); Task task = tasks.Values.GetEnumerator().Next(); app.WaitForState(task, TaskState.Scheduled); IDictionary<TaskAttemptId, TaskAttempt> attempts = tasks.Values.GetEnumerator().Next ().GetAttempts(); NUnit.Framework.Assert.AreEqual("Num attempts is not correct", maxAttempts, attempts .Count); TaskAttempt attempt = attempts.Values.GetEnumerator().Next(); app.WaitForInternalState((TaskAttemptImpl)attempt, TaskAttemptStateInternal.Assigned ); app.WaitForState(job, JobState.Failed); string diagnostics = attempt.GetDiagnostics().ToString(); Log.Info("attempt.getDiagnostics: " + diagnostics); NUnit.Framework.Assert.IsTrue(diagnostics.Contains("Container launch failed for " + "container_0_0000_01_000000 : ")); NUnit.Framework.Assert.IsTrue(diagnostics.Contains("java.net.SocketTimeoutException: 3000 millis timeout while waiting for channel" )); } finally { server.Stop(); app.Stop(); } }
public virtual void TestEventsFlushOnStop() { Configuration conf = new Configuration(); MRApp app = new TestJobHistoryEvents.MRAppWithSpecialHistoryHandler(1, 0, true, this .GetType().FullName, true); app.Submit(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); JobId jobId = job.GetID(); Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString()); app.WaitForState(job, JobState.Succeeded); // make sure all events are flushed app.WaitForState(Service.STATE.Stopped); /* * Use HistoryContext to read logged events and verify the number of * completed maps */ HistoryContext context = new JobHistory(); ((JobHistory)context).Init(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job parsedJob = context.GetJob(jobId); NUnit.Framework.Assert.AreEqual("CompletedMaps not correct", 1, parsedJob.GetCompletedMaps ()); IDictionary <TaskId, Task> tasks = parsedJob.GetTasks(); NUnit.Framework.Assert.AreEqual("No of tasks not correct", 1, tasks.Count); VerifyTask(tasks.Values.GetEnumerator().Next()); IDictionary <TaskId, Task> maps = parsedJob.GetTasks(TaskType.Map); NUnit.Framework.Assert.AreEqual("No of maps not correct", 1, maps.Count); NUnit.Framework.Assert.AreEqual("Job state not currect", JobState.Succeeded, parsedJob .GetState()); }
private void GetCounters(AppContext ctx, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job) { if (job == null) { return; } total = job.GetAllCounters(); bool needTotalCounters = false; if (total == null) { total = new Counters(); needTotalCounters = true; } map = new Counters(); reduce = new Counters(); // Get all types of counters IDictionary <TaskId, Task> tasks = job.GetTasks(); foreach (Task t in tasks.Values) { Counters counters = t.GetCounters(); if (counters == null) { continue; } switch (t.GetType()) { case TaskType.Map: { map.IncrAllCounters(counters); break; } case TaskType.Reduce: { reduce.IncrAllCounters(counters); break; } } if (needTotalCounters) { total.IncrAllCounters(counters); } } }
public virtual void TestSingleCounterView() { AppContext appContext = new MockAppContext(0, 1, 1, 1); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = appContext.GetAllJobs().Values.GetEnumerator ().Next(); // add a failed task to the job without any counters Task failedTask = MockJobs.NewTask(job.GetID(), 2, 1, true); IDictionary <TaskId, Task> tasks = job.GetTasks(); tasks[failedTask.GetID()] = failedTask; IDictionary <string, string> @params = GetJobParams(appContext); @params[AMParams.CounterGroup] = "org.apache.hadoop.mapreduce.FileSystemCounter"; @params[AMParams.CounterName] = "HDFS_WRITE_OPS"; WebAppTests.TestPage <AppContext>(typeof(SingleCounterPage), appContext, @params); }
public virtual void TestContainerPassThrough() { MRApp app = new _MRApp_494(this, 0, 1, true, this.GetType().FullName, true); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration()); app.WaitForState(job, JobState.Succeeded); app.VerifyCompleted(); ICollection <Task> tasks = job.GetTasks().Values; ICollection <TaskAttempt> taskAttempts = tasks.GetEnumerator().Next().GetAttempts( ).Values; TaskAttemptImpl taskAttempt = (TaskAttemptImpl)taskAttempts.GetEnumerator().Next( ); // Container from RM should pass through to the launcher. Container object // should be the same. NUnit.Framework.Assert.IsTrue(taskAttempt.container == containerObtainedByContainerLauncher ); }
/// <exception cref="System.IO.IOException"/> public virtual GetTaskReportsResponse GetTaskReports(GetTaskReportsRequest request ) { JobId jobId = request.GetJobId(); TaskType taskType = request.GetTaskType(); GetTaskReportsResponse response = this.recordFactory.NewRecordInstance <GetTaskReportsResponse >(); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = this.VerifyAndGetJob(jobId, true ); ICollection <Task> tasks = job.GetTasks(taskType).Values; foreach (Task task in tasks) { response.AddTaskReport(task.GetReport()); } return(response); }
public virtual void TestAverageReduceTime() { Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <CompletedJob >(); Task task1 = Org.Mockito.Mockito.Mock <Task>(); Task task2 = Org.Mockito.Mockito.Mock <Task>(); JobId jobId = MRBuilderUtils.NewJobId(1L, 1, 1); TaskId taskId1 = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Reduce); TaskId taskId2 = MRBuilderUtils.NewTaskId(jobId, 2, TaskType.Reduce); TaskAttemptId taskAttemptId1 = MRBuilderUtils.NewTaskAttemptId(taskId1, 1); TaskAttemptId taskAttemptId2 = MRBuilderUtils.NewTaskAttemptId(taskId2, 2); TaskAttempt taskAttempt1 = Org.Mockito.Mockito.Mock <TaskAttempt>(); TaskAttempt taskAttempt2 = Org.Mockito.Mockito.Mock <TaskAttempt>(); JobReport jobReport = Org.Mockito.Mockito.Mock <JobReport>(); Org.Mockito.Mockito.When(taskAttempt1.GetState()).ThenReturn(TaskAttemptState.Succeeded ); Org.Mockito.Mockito.When(taskAttempt1.GetLaunchTime()).ThenReturn(0L); Org.Mockito.Mockito.When(taskAttempt1.GetShuffleFinishTime()).ThenReturn(4L); Org.Mockito.Mockito.When(taskAttempt1.GetSortFinishTime()).ThenReturn(6L); Org.Mockito.Mockito.When(taskAttempt1.GetFinishTime()).ThenReturn(8L); Org.Mockito.Mockito.When(taskAttempt2.GetState()).ThenReturn(TaskAttemptState.Succeeded ); Org.Mockito.Mockito.When(taskAttempt2.GetLaunchTime()).ThenReturn(5L); Org.Mockito.Mockito.When(taskAttempt2.GetShuffleFinishTime()).ThenReturn(10L); Org.Mockito.Mockito.When(taskAttempt2.GetSortFinishTime()).ThenReturn(22L); Org.Mockito.Mockito.When(taskAttempt2.GetFinishTime()).ThenReturn(42L); Org.Mockito.Mockito.When(task1.GetType()).ThenReturn(TaskType.Reduce); Org.Mockito.Mockito.When(task2.GetType()).ThenReturn(TaskType.Reduce); Org.Mockito.Mockito.When(task1.GetAttempts()).ThenReturn(new _Dictionary_120(taskAttemptId1 , taskAttempt1)); Org.Mockito.Mockito.When(task2.GetAttempts()).ThenReturn(new _Dictionary_123(taskAttemptId2 , taskAttempt2)); Org.Mockito.Mockito.When(job.GetTasks()).ThenReturn(new _Dictionary_127(taskId1, task1, taskId2, task2)); Org.Mockito.Mockito.When(job.GetID()).ThenReturn(jobId); Org.Mockito.Mockito.When(job.GetReport()).ThenReturn(jobReport); Org.Mockito.Mockito.When(job.GetName()).ThenReturn("TestJobInfo"); Org.Mockito.Mockito.When(job.GetState()).ThenReturn(JobState.Succeeded); JobInfo jobInfo = new JobInfo(job); NUnit.Framework.Assert.AreEqual(11L, jobInfo.GetAvgReduceTime()); }
public virtual void Setup() { this.conf = new JobConf(); this.conf.Set(CommonConfigurationKeys.HadoopSecurityGroupMapping, typeof(TestHsWebServicesAcls.NullGroupsProvider ).FullName); this.conf.SetBoolean(MRConfig.MrAclsEnabled, true); Groups.GetUserToGroupsMappingService(conf); this.ctx = BuildHistoryContext(this.conf); WebApp webApp = Org.Mockito.Mockito.Mock <HsWebApp>(); Org.Mockito.Mockito.When(webApp.Name()).ThenReturn("hsmockwebapp"); this.hsWebServices = new HsWebServices(ctx, conf, webApp); this.hsWebServices.SetResponse(Org.Mockito.Mockito.Mock <HttpServletResponse>()); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = ctx.GetAllJobs().Values.GetEnumerator ().Next(); this.jobIdStr = job.GetID().ToString(); Task task = job.GetTasks().Values.GetEnumerator().Next(); this.taskIdStr = task.GetID().ToString(); this.taskAttemptIdStr = task.GetAttempts().Keys.GetEnumerator().Next().ToString(); }
public virtual void VerifyAMTaskXML(NodeList nodes, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job) { NUnit.Framework.Assert.AreEqual("incorrect number of elements", 2, nodes.GetLength ()); foreach (Task task in job.GetTasks().Values) { TaskId id = task.GetID(); string tid = MRApps.ToString(id); bool found = false; for (int i = 0; i < nodes.GetLength(); i++) { Element element = (Element)nodes.Item(i); if (tid.Matches(WebServicesTestUtils.GetXmlString(element, "id"))) { found = true; VerifyAMSingleTaskXML(element, task); } } NUnit.Framework.Assert.IsTrue("task with id: " + tid + " not in web service output" , found); } }
/* * (non-Javadoc) * @see org.apache.hadoop.mapreduce.v2.hs.webapp.HsTaskPage.AttemptsBlock#getTaskAttempts() * @return the attempts that are for a given job and a specific type/state. */ protected internal override ICollection <TaskAttempt> GetTaskAttempts() { IList <TaskAttempt> fewTaskAttemps = new AList <TaskAttempt>(); string taskTypeStr = $(AMParams.TaskType); TaskType taskType = MRApps.TaskType(taskTypeStr); string attemptStateStr = $(AMParams.AttemptState); MRApps.TaskAttemptStateUI neededState = MRApps.TaskAttemptState(attemptStateStr); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job j = app.GetJob(); IDictionary <TaskId, Task> tasks = j.GetTasks(taskType); foreach (Task task in tasks.Values) { IDictionary <TaskAttemptId, TaskAttempt> attempts = task.GetAttempts(); foreach (TaskAttempt attempt in attempts.Values) { if (neededState.CorrespondsTo(attempt.GetState())) { fewTaskAttemps.AddItem(attempt); } } } return(fewTaskAttemps); }