public virtual void TestNotificationOnLastRetryUnregistrationFailure() { HttpServer2 server = StartHttpServer(); MRApp app = Org.Mockito.Mockito.Spy(new TestJobEndNotifier.MRAppWithCustomContainerAllocator (this, 2, 2, false, this.GetType().FullName, true, 2, false)); // Currently, we will have isLastRetry always equals to false at beginning // of MRAppMaster, except staging area exists or commit already started at // the beginning. // Now manually set isLastRetry to true and this should reset to false when // unregister failed. app.isLastAMRetry = true; Org.Mockito.Mockito.DoNothing().When(app).Sysexit(); JobConf conf = new JobConf(); conf.Set(JobContext.MrJobEndNotificationUrl, TestJobEndNotifier.JobEndServlet.baseUrl + "jobend?jobid=$jobId&status=$jobStatus"); JobImpl job = (JobImpl)app.Submit(conf); app.WaitForState(job, JobState.Running); app.GetContext().GetEventHandler().Handle(new JobEvent(app.GetJobId(), JobEventType .JobAmReboot)); app.WaitForInternalState(job, JobStateInternal.Reboot); // Now shutdown. User should see FAILED state. // Unregistration fails: isLastAMRetry is recalculated, this is ///reboot will stop service internally, we don't need to shutdown twice app.WaitForServiceToStop(10000); NUnit.Framework.Assert.IsFalse(app.IsLastAMRetry()); // Since it's not last retry, JobEndServlet didn't called NUnit.Framework.Assert.AreEqual(0, TestJobEndNotifier.JobEndServlet.calledTimes); NUnit.Framework.Assert.IsNull(TestJobEndNotifier.JobEndServlet.requestUri); NUnit.Framework.Assert.IsNull(TestJobEndNotifier.JobEndServlet.foundJobState); server.Stop(); }
public virtual void TestNotificationOnLastRetryNormalShutdown() { HttpServer2 server = StartHttpServer(); // Act like it is the second attempt. Default max attempts is 2 MRApp app = Org.Mockito.Mockito.Spy(new TestJobEndNotifier.MRAppWithCustomContainerAllocator (this, 2, 2, true, this.GetType().FullName, true, 2, true)); Org.Mockito.Mockito.DoNothing().When(app).Sysexit(); JobConf conf = new JobConf(); conf.Set(JobContext.MrJobEndNotificationUrl, TestJobEndNotifier.JobEndServlet.baseUrl + "jobend?jobid=$jobId&status=$jobStatus"); JobImpl job = (JobImpl)app.Submit(conf); app.WaitForInternalState(job, JobStateInternal.Succeeded); // Unregistration succeeds: successfullyUnregistered is set app.ShutDownJob(); NUnit.Framework.Assert.IsTrue(app.IsLastAMRetry()); NUnit.Framework.Assert.AreEqual(1, TestJobEndNotifier.JobEndServlet.calledTimes); NUnit.Framework.Assert.AreEqual("jobid=" + job.GetID() + "&status=SUCCEEDED", TestJobEndNotifier.JobEndServlet .requestUri.GetQuery()); NUnit.Framework.Assert.AreEqual(JobState.Succeeded.ToString(), TestJobEndNotifier.JobEndServlet .foundJobState); server.Stop(); }
public virtual void TestAbsentNotificationOnNotLastRetryUnregistrationFailure() { HttpServer2 server = StartHttpServer(); MRApp app = Org.Mockito.Mockito.Spy(new TestJobEndNotifier.MRAppWithCustomContainerAllocator (this, 2, 2, false, this.GetType().FullName, true, 1, false)); Org.Mockito.Mockito.DoNothing().When(app).Sysexit(); JobConf conf = new JobConf(); conf.Set(JobContext.MrJobEndNotificationUrl, TestJobEndNotifier.JobEndServlet.baseUrl + "jobend?jobid=$jobId&status=$jobStatus"); JobImpl job = (JobImpl)app.Submit(conf); app.WaitForState(job, JobState.Running); app.GetContext().GetEventHandler().Handle(new JobEvent(app.GetJobId(), JobEventType .JobAmReboot)); app.WaitForInternalState(job, JobStateInternal.Reboot); // Now shutdown. // Unregistration fails: isLastAMRetry is recalculated, this is not app.ShutDownJob(); // Not the last AM attempt. So user should that the job is still running. app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.IsFalse(app.IsLastAMRetry()); NUnit.Framework.Assert.AreEqual(0, TestJobEndNotifier.JobEndServlet.calledTimes); NUnit.Framework.Assert.IsNull(TestJobEndNotifier.JobEndServlet.requestUri); NUnit.Framework.Assert.IsNull(TestJobEndNotifier.JobEndServlet.foundJobState); server.Stop(); }
public virtual void TestCommitPending() { MRApp app = new MRApp(1, 0, false, this.GetType().FullName, true); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration()); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task task = it.Next(); app.WaitForState(task, TaskState.Running); TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next(); app.WaitForState(attempt, TaskAttemptState.Running); //send the commit pending signal to the task app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(attempt.GetID(), TaskAttemptEventType .TaCommitPending)); //wait for first attempt to commit pending app.WaitForState(attempt, TaskAttemptState.CommitPending); //re-send the commit pending signal to the task app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(attempt.GetID(), TaskAttemptEventType .TaCommitPending)); //the task attempt should be still at COMMIT_PENDING app.WaitForState(attempt, TaskAttemptState.CommitPending); //send the done signal to the task app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task.GetAttempts() .Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone)); app.WaitForState(job, JobState.Succeeded); }
public virtual void TestZeroMapReduces() { MRApp app = new MRApp(0, 0, true, this.GetType().FullName, true); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration()); app.WaitForState(job, JobState.Succeeded); }
public virtual void TestMapReduce() { MRApp app = new MRApp(2, 2, true, this.GetType().FullName, true); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration()); app.WaitForState(job, JobState.Succeeded); app.VerifyCompleted(); NUnit.Framework.Assert.AreEqual(Runtime.GetProperty("user.name"), job.GetUserName ()); }
public virtual void TestJobSuccess() { MRApp app = new MRApp(2, 2, true, this.GetType().FullName, true, false); JobImpl job = (JobImpl)app.Submit(new Configuration()); app.WaitForInternalState(job, JobStateInternal.Succeeded); // AM is not unregistered NUnit.Framework.Assert.AreEqual(JobState.Running, job.GetState()); // imitate that AM is unregistered app.successfullyUnregistered.Set(true); app.WaitForState(job, JobState.Succeeded); }
public TestJob(MRApp _enclosing, JobId jobId, ApplicationAttemptId applicationAttemptId , Configuration conf, EventHandler eventHandler, TaskAttemptListener taskAttemptListener , Clock clock, OutputCommitter committer, bool newApiCommitter, string user, AppContext appContext, JobStateInternal forcedState, string diagnostic) : base(jobId, MRApp.GetApplicationAttemptId(MRApp.applicationId, this._enclosing. GetStartCount()), conf, eventHandler, taskAttemptListener, new JobTokenSecretManager (), new Credentials(), clock, this._enclosing.GetCompletedTaskFromPreviousRun(), this._enclosing.metrics, committer, newApiCommitter, user, Runtime.CurrentTimeMillis (), this._enclosing.GetAllAMInfos(), appContext, forcedState, diagnostic) { this._enclosing = _enclosing; // This "this leak" is okay because the retained pointer is in an // instance variable. this.localStateMachine = this.localFactory.Make(this); }
/// <summary>Runs memory and time benchmark with Mock MRApp.</summary> /// <exception cref="System.Exception"/> public virtual void Run(MRApp app) { Logger rootLogger = LogManager.GetRootLogger(); rootLogger.SetLevel(Level.Warn); long startTime = Runtime.CurrentTimeMillis(); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration()); while (!job.GetReport().GetJobState().Equals(JobState.Succeeded)) { PrintStat(job, startTime); Sharpen.Thread.Sleep(2000); } PrintStat(job, startTime); }
public virtual void TestJobError() { MRApp app = new MRApp(1, 0, false, this.GetType().FullName, true); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration()); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task task = it.Next(); app.WaitForState(task, TaskState.Running); //send an invalid event on task at current state app.GetContext().GetEventHandler().Handle(new TaskEvent(task.GetID(), TaskEventType .TSchedule)); //this must lead to job error app.WaitForState(job, JobState.Error); }
//@Test /// <exception cref="System.Exception"/> public virtual void TestCompletedMapsForReduceSlowstart() { MRApp app = new MRApp(2, 1, false, this.GetType().FullName, true); Configuration conf = new Configuration(); //after half of the map completion, reduce will start conf.SetFloat(MRJobConfig.CompletedMapsForReduceSlowstart, 0.5f); //uberization forces full slowstart (1.0), so disable that conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Running); //all maps would be running NUnit.Framework.Assert.AreEqual("Num tasks not correct", 3, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task mapTask1 = it.Next(); Task mapTask2 = it.Next(); Task reduceTask = it.Next(); // all maps must be running app.WaitForState(mapTask1, TaskState.Running); app.WaitForState(mapTask2, TaskState.Running); TaskAttempt task1Attempt = mapTask1.GetAttempts().Values.GetEnumerator().Next(); TaskAttempt task2Attempt = mapTask2.GetAttempts().Values.GetEnumerator().Next(); //before sending the TA_DONE, event make sure attempt has come to //RUNNING state app.WaitForState(task1Attempt, TaskAttemptState.Running); app.WaitForState(task2Attempt, TaskAttemptState.Running); // reduces must be in NEW state NUnit.Framework.Assert.AreEqual("Reduce Task state not correct", TaskState.New, reduceTask .GetReport().GetTaskState()); //send the done signal to the 1st map task app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapTask1.GetAttempts ().Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone)); //wait for first map task to complete app.WaitForState(mapTask1, TaskState.Succeeded); //Once the first map completes, it will schedule the reduces //now reduce must be running app.WaitForState(reduceTask, TaskState.Running); //send the done signal to 2nd map and the reduce to complete the job app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapTask2.GetAttempts ().Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone)); app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(reduceTask.GetAttempts ().Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone)); app.WaitForState(job, JobState.Succeeded); }
public virtual void TestJobRebootNotLastRetryOnUnregistrationFailure() { MRApp app = new MRApp(1, 0, false, this.GetType().FullName, true); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration()); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task task = it.Next(); app.WaitForState(task, TaskState.Running); //send an reboot event app.GetContext().GetEventHandler().Handle(new JobEvent(job.GetID(), JobEventType. JobAmReboot)); // return exteranl state as RUNNING since otherwise the JobClient will // prematurely exit. app.WaitForState(job, JobState.Running); }
private void UpdateStatus(MRApp app, TaskAttempt attempt, Phase phase) { TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus (); status.counters = new Counters(); status.fetchFailedMaps = new AList <TaskAttemptId>(); status.id = attempt.GetID(); status.mapFinishTime = 0; status.phase = phase; status.progress = 0.5f; status.shuffleFinishTime = 0; status.sortFinishTime = 0; status.stateString = "OK"; status.taskState = attempt.GetState(); TaskAttemptStatusUpdateEvent @event = new TaskAttemptStatusUpdateEvent(attempt.GetID (), status); app.GetContext().GetEventHandler().Handle(@event); }
public virtual void Handle(ContainerAllocatorEvent @event) { ContainerId cId = ContainerId.NewContainerId(this._enclosing.GetContext().GetApplicationAttemptId (), this.containerCount++); NodeId nodeId = NodeId.NewInstance(MRApp.NmHost, MRApp.NmPort); Resource resource = Resource.NewInstance(1234, 2); ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier( cId, nodeId.ToString(), "user", resource, Runtime.CurrentTimeMillis() + 10000, 42 , 42, Priority.NewInstance(0), 0); Token containerToken = MRApp.NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString ("password"), containerTokenIdentifier); Container container = Container.NewInstance(cId, nodeId, MRApp.NmHost + ":" + MRApp .NmHttpPort, resource, null, containerToken); JobID id = TypeConverter.FromYarn(MRApp.applicationId); JobId jobId = TypeConverter.ToYarn(id); this._enclosing.GetContext().GetEventHandler().Handle(new JobHistoryEvent(jobId, new NormalizedResourceEvent(TaskType.Reduce, 100))); this._enclosing.GetContext().GetEventHandler().Handle(new JobHistoryEvent(jobId, new NormalizedResourceEvent(TaskType.Map, 100))); this._enclosing.GetContext().GetEventHandler().Handle(new TaskAttemptContainerAssignedEvent (@event.GetAttemptID(), container, null)); }
public virtual void TestJobRebootOnLastRetryOnUnregistrationFailure() { // make startCount as 2 since this is last retry which equals to // DEFAULT_MAX_AM_RETRY // The last param mocks the unregistration failure MRApp app = new MRApp(1, 0, false, this.GetType().FullName, true, 2, false); Configuration conf = new Configuration(); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task task = it.Next(); app.WaitForState(task, TaskState.Running); //send an reboot event app.GetContext().GetEventHandler().Handle(new JobEvent(job.GetID(), JobEventType. JobAmReboot)); app.WaitForInternalState((JobImpl)job, JobStateInternal.Reboot); // return exteranl state as RUNNING if this is the last retry while // unregistration fails app.WaitForState(job, JobState.Running); }
public virtual void TestFetchFailureMultipleReduces() { MRApp app = new MRApp(1, 3, false, this.GetType().FullName, true); Configuration conf = new Configuration(); // map -> reduce -> fetch-failure -> map retry is incompatible with // sequential, single-task-attempt approach in uber-AM, so disable: conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Running); //all maps would be running NUnit.Framework.Assert.AreEqual("Num tasks not correct", 4, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task mapTask = it.Next(); Task reduceTask = it.Next(); Task reduceTask2 = it.Next(); Task reduceTask3 = it.Next(); //wait for Task state move to RUNNING app.WaitForState(mapTask, TaskState.Running); TaskAttempt mapAttempt1 = mapTask.GetAttempts().Values.GetEnumerator().Next(); app.WaitForState(mapAttempt1, TaskAttemptState.Running); //send the done signal to the map attempt app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapAttempt1.GetID( ), TaskAttemptEventType.TaDone)); // wait for map success app.WaitForState(mapTask, TaskState.Succeeded); TaskAttemptCompletionEvent[] events = job.GetTaskAttemptCompletionEvents(0, 100); NUnit.Framework.Assert.AreEqual("Num completion events not correct", 1, events.Length ); NUnit.Framework.Assert.AreEqual("Event status not correct", TaskAttemptCompletionEventStatus .Succeeded, events[0].GetStatus()); // wait for reduce to start running app.WaitForState(reduceTask, TaskState.Running); app.WaitForState(reduceTask2, TaskState.Running); app.WaitForState(reduceTask3, TaskState.Running); TaskAttempt reduceAttempt = reduceTask.GetAttempts().Values.GetEnumerator().Next( ); app.WaitForState(reduceAttempt, TaskAttemptState.Running); UpdateStatus(app, reduceAttempt, Phase.Shuffle); TaskAttempt reduceAttempt2 = reduceTask2.GetAttempts().Values.GetEnumerator().Next (); app.WaitForState(reduceAttempt2, TaskAttemptState.Running); UpdateStatus(app, reduceAttempt2, Phase.Shuffle); TaskAttempt reduceAttempt3 = reduceTask3.GetAttempts().Values.GetEnumerator().Next (); app.WaitForState(reduceAttempt3, TaskAttemptState.Running); UpdateStatus(app, reduceAttempt3, Phase.Shuffle); //send 2 fetch failures from reduce to prepare for map re execution SendFetchFailure(app, reduceAttempt, mapAttempt1); SendFetchFailure(app, reduceAttempt, mapAttempt1); //We should not re-launch the map task yet NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, mapTask.GetState()); UpdateStatus(app, reduceAttempt2, Phase.Reduce); UpdateStatus(app, reduceAttempt3, Phase.Reduce); //send 3rd fetch failures from reduce to trigger map re execution SendFetchFailure(app, reduceAttempt, mapAttempt1); //wait for map Task state move back to RUNNING app.WaitForState(mapTask, TaskState.Running); //map attempt must have become FAILED NUnit.Framework.Assert.AreEqual("Map TaskAttempt state not correct", TaskAttemptState .Failed, mapAttempt1.GetState()); NUnit.Framework.Assert.AreEqual("Num attempts in Map Task not correct", 2, mapTask .GetAttempts().Count); IEnumerator <TaskAttempt> atIt = mapTask.GetAttempts().Values.GetEnumerator(); atIt.Next(); TaskAttempt mapAttempt2 = atIt.Next(); app.WaitForState(mapAttempt2, TaskAttemptState.Running); //send the done signal to the second map attempt app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(mapAttempt2.GetID( ), TaskAttemptEventType.TaDone)); // wait for map success app.WaitForState(mapTask, TaskState.Succeeded); //send done to reduce app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(reduceAttempt.GetID (), TaskAttemptEventType.TaDone)); //send done to reduce app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(reduceAttempt2.GetID (), TaskAttemptEventType.TaDone)); //send done to reduce app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(reduceAttempt3.GetID (), TaskAttemptEventType.TaDone)); app.WaitForState(job, JobState.Succeeded); //previous completion event now becomes obsolete NUnit.Framework.Assert.AreEqual("Event status not correct", TaskAttemptCompletionEventStatus .Obsolete, events[0].GetStatus()); events = job.GetTaskAttemptCompletionEvents(0, 100); NUnit.Framework.Assert.AreEqual("Num completion events not correct", 6, events.Length ); NUnit.Framework.Assert.AreEqual("Event map attempt id not correct", mapAttempt1.GetID (), events[0].GetAttemptId()); NUnit.Framework.Assert.AreEqual("Event map attempt id not correct", mapAttempt1.GetID (), events[1].GetAttemptId()); NUnit.Framework.Assert.AreEqual("Event map attempt id not correct", mapAttempt2.GetID (), events[2].GetAttemptId()); NUnit.Framework.Assert.AreEqual("Event reduce attempt id not correct", reduceAttempt .GetID(), events[3].GetAttemptId()); NUnit.Framework.Assert.AreEqual("Event status not correct for map attempt1", TaskAttemptCompletionEventStatus .Obsolete, events[0].GetStatus()); NUnit.Framework.Assert.AreEqual("Event status not correct for map attempt1", TaskAttemptCompletionEventStatus .Failed, events[1].GetStatus()); NUnit.Framework.Assert.AreEqual("Event status not correct for map attempt2", TaskAttemptCompletionEventStatus .Succeeded, events[2].GetStatus()); NUnit.Framework.Assert.AreEqual("Event status not correct for reduce attempt1", TaskAttemptCompletionEventStatus .Succeeded, events[3].GetStatus()); TaskCompletionEvent[] mapEvents = job.GetMapAttemptCompletionEvents(0, 2); TaskCompletionEvent[] convertedEvents = TypeConverter.FromYarn(events); NUnit.Framework.Assert.AreEqual("Incorrect number of map events", 2, mapEvents.Length ); Assert.AssertArrayEquals("Unexpected map events", Arrays.CopyOfRange(convertedEvents , 0, 2), mapEvents); mapEvents = job.GetMapAttemptCompletionEvents(2, 200); NUnit.Framework.Assert.AreEqual("Incorrect number of map events", 1, mapEvents.Length ); NUnit.Framework.Assert.AreEqual("Unexpected map event", convertedEvents[2], mapEvents [0]); }
internal MRAppContainerAllocator(MRApp _enclosing) { this._enclosing = _enclosing; }
public MockContainerLauncher(MRApp _enclosing) { this._enclosing = _enclosing; }
private void SendFetchFailure(MRApp app, TaskAttempt reduceAttempt, TaskAttempt mapAttempt ) { app.GetContext().GetEventHandler().Handle(new JobTaskAttemptFetchFailureEvent(reduceAttempt .GetID(), Arrays.AsList(new TaskAttemptId[] { mapAttempt.GetID() }))); }
public _EventHandler_473(MRApp _enclosing) { this._enclosing = _enclosing; }