protected override void Dispatch(Org.Apache.Hadoop.Yarn.Event.Event @event) { if (@event is TaskAttemptEvent) { TaskAttemptEvent killEvent = (TaskAttemptEvent)@event; if (killEvent.GetType() == TaskAttemptEventType.TaKill) { TaskAttemptId taID = killEvent.GetTaskAttemptID(); if (taID.GetTaskId().GetTaskType() == TaskType.Reduce && taID.GetTaskId().GetId() == 0 && taID.GetId() == 0) { base.Dispatch(new TaskAttemptEvent(taID, TaskAttemptEventType.TaDone)); base.Dispatch(new TaskAttemptEvent(taID, TaskAttemptEventType.TaContainerCleaned) ); base.Dispatch(new TaskTAttemptEvent(taID, TaskEventType.TAttemptSucceeded)); this.cachedKillEvent = killEvent; return; } } } else { if (@event is TaskEvent) { TaskEvent taskEvent = (TaskEvent)@event; if (taskEvent.GetType() == TaskEventType.TAttemptSucceeded && this.cachedKillEvent != null) { base.Dispatch(this.cachedKillEvent); return; } } } base.Dispatch(@event); }
public virtual void Handle(Org.Apache.Hadoop.Yarn.Event.Event @event) { lock (this) { this.@event = @event; Sharpen.Runtime.NotifyAll(this); } }
protected override void Dispatch(Org.Apache.Hadoop.Yarn.Event.Event @event) { if (@event.GetType() == RMFatalEventType.TransitionToActiveFailed) { this.eventreceived++; } else { base.Dispatch(@event); } }
public void Handle(Org.Apache.Hadoop.Yarn.Event.Event @event) { TestLocalContainerLauncher.Log.Info("handling event " + @event.GetType() + " with type " + @event.GetType()); if (@event is TaskAttemptEvent) { if (@event.GetType() == TaskAttemptEventType.TaContainerCleaned) { isDone.CountDown(); } } }
public virtual void TestBasic() { AppContext mockContext = Org.Mockito.Mockito.Mock <AppContext>(); OutputCommitter mockCommitter = Org.Mockito.Mockito.Mock <OutputCommitter>(); Clock mockClock = Org.Mockito.Mockito.Mock <Clock>(); CommitterEventHandler handler = new CommitterEventHandler(mockContext, mockCommitter , new TestCommitterEventHandler.TestingRMHeartbeatHandler()); YarnConfiguration conf = new YarnConfiguration(); conf.Set(MRJobConfig.MrAmStagingDir, stagingDir); JobContext mockJobContext = Org.Mockito.Mockito.Mock <JobContext>(); ApplicationAttemptId attemptid = ConverterUtils.ToApplicationAttemptId("appattempt_1234567890000_0001_0" ); JobId jobId = TypeConverter.ToYarn(TypeConverter.FromYarn(attemptid.GetApplicationId ())); TestCommitterEventHandler.WaitForItHandler waitForItHandler = new TestCommitterEventHandler.WaitForItHandler (); Org.Mockito.Mockito.When(mockContext.GetApplicationID()).ThenReturn(attemptid.GetApplicationId ()); Org.Mockito.Mockito.When(mockContext.GetApplicationAttemptId()).ThenReturn(attemptid ); Org.Mockito.Mockito.When(mockContext.GetEventHandler()).ThenReturn(waitForItHandler ); Org.Mockito.Mockito.When(mockContext.GetClock()).ThenReturn(mockClock); handler.Init(conf); handler.Start(); try { handler.Handle(new CommitterJobCommitEvent(jobId, mockJobContext)); string user = UserGroupInformation.GetCurrentUser().GetShortUserName(); Path startCommitFile = MRApps.GetStartJobCommitFile(conf, user, jobId); Path endCommitSuccessFile = MRApps.GetEndJobCommitSuccessFile(conf, user, jobId); Path endCommitFailureFile = MRApps.GetEndJobCommitFailureFile(conf, user, jobId); Org.Apache.Hadoop.Yarn.Event.Event e = waitForItHandler.GetAndClearEvent(); NUnit.Framework.Assert.IsNotNull(e); NUnit.Framework.Assert.IsTrue(e is JobCommitCompletedEvent); FileSystem fs = FileSystem.Get(conf); NUnit.Framework.Assert.IsTrue(startCommitFile.ToString(), fs.Exists(startCommitFile )); NUnit.Framework.Assert.IsTrue(endCommitSuccessFile.ToString(), fs.Exists(endCommitSuccessFile )); NUnit.Framework.Assert.IsFalse(endCommitFailureFile.ToString(), fs.Exists(endCommitFailureFile )); Org.Mockito.Mockito.Verify(mockCommitter).CommitJob(Matchers.Any <JobContext>()); } finally { handler.Stop(); } }
protected override void Dispatch(Org.Apache.Hadoop.Yarn.Event.Event @event) { if (@event is WritingApplicationHistoryEvent) { WritingApplicationHistoryEvent ashEvent = (WritingApplicationHistoryEvent)@event; switch (ashEvent.GetType()) { case WritingHistoryEventType.AppStart: { IncrementCounts(((WritingApplicationStartEvent)@event).GetApplicationId()); break; } case WritingHistoryEventType.AppFinish: { IncrementCounts(((WritingApplicationFinishEvent)@event).GetApplicationId()); break; } case WritingHistoryEventType.AppAttemptStart: { IncrementCounts(((WritingApplicationAttemptStartEvent)@event).GetApplicationAttemptId ().GetApplicationId()); break; } case WritingHistoryEventType.AppAttemptFinish: { IncrementCounts(((WritingApplicationAttemptFinishEvent)@event).GetApplicationAttemptId ().GetApplicationId()); break; } case WritingHistoryEventType.ContainerStart: { IncrementCounts(((WritingContainerStartEvent)@event).GetContainerId().GetApplicationAttemptId ().GetApplicationId()); break; } case WritingHistoryEventType.ContainerFinish: { IncrementCounts(((WritingContainerFinishEvent)@event).GetContainerId().GetApplicationAttemptId ().GetApplicationId()); break; } } } base.Dispatch(@event); }
/// <exception cref="System.Exception"/> public virtual Org.Apache.Hadoop.Yarn.Event.Event GetAndClearEvent() { lock (this) { if (@event == null) { long waitTime = 5000; long waitStartTime = Time.MonotonicNow(); while (@event == null && Time.MonotonicNow() - waitStartTime < waitTime) { //Wait for at most 5 sec Sharpen.Runtime.Wait(this, waitTime); } } Org.Apache.Hadoop.Yarn.Event.Event e = @event; @event = null; return(e); } }
protected override void Dispatch(Org.Apache.Hadoop.Yarn.Event.Event @event) { if (@event is TaskAttemptEvent) { TaskAttemptEvent attemptEvent = (TaskAttemptEvent)@event; TaskAttemptId attemptID = ((TaskAttemptEvent)@event).GetTaskAttemptID(); if (attemptEvent.GetType() == this.attemptEventTypeToWait && attemptID.GetTaskId( ).GetId() == 0 && attemptID.GetId() == 0) { try { latch.Await(); } catch (Exception e) { Sharpen.Runtime.PrintStackTrace(e); } } } base.Dispatch(@event); }
public void Handle(Org.Apache.Hadoop.Yarn.Event.Event @event) { }
/// <exception cref="System.Exception"/> public virtual void TestContainerCleaned() { Log.Info("STARTING testContainerCleaned"); CyclicBarrier startLaunchBarrier = new CyclicBarrier(2); CyclicBarrier completeLaunchBarrier = new CyclicBarrier(2); AppContext mockContext = Org.Mockito.Mockito.Mock <AppContext>(); EventHandler mockEventHandler = Org.Mockito.Mockito.Mock <EventHandler>(); Org.Mockito.Mockito.When(mockContext.GetEventHandler()).ThenReturn(mockEventHandler ); TestContainerLauncherImpl.ContainerManagementProtocolClient mockCM = new TestContainerLauncherImpl.ContainerManagerForTest (startLaunchBarrier, completeLaunchBarrier); TestContainerLauncherImpl.ContainerLauncherImplUnderTest ut = new TestContainerLauncherImpl.ContainerLauncherImplUnderTest (mockContext, mockCM); Configuration conf = new Configuration(); ut.Init(conf); ut.Start(); try { ContainerId contId = MakeContainerId(0l, 0, 0, 1); TaskAttemptId taskAttemptId = MakeTaskAttemptId(0l, 0, 0, TaskType.Map, 0); string cmAddress = "127.0.0.1:8000"; StartContainersResponse startResp = recordFactory.NewRecordInstance <StartContainersResponse >(); startResp.SetAllServicesMetaData(serviceResponse); Log.Info("inserting launch event"); ContainerRemoteLaunchEvent mockLaunchEvent = Org.Mockito.Mockito.Mock <ContainerRemoteLaunchEvent >(); Org.Mockito.Mockito.When(mockLaunchEvent.GetType()).ThenReturn(ContainerLauncher.EventType .ContainerRemoteLaunch); Org.Mockito.Mockito.When(mockLaunchEvent.GetContainerID()).ThenReturn(contId); Org.Mockito.Mockito.When(mockLaunchEvent.GetTaskAttemptID()).ThenReturn(taskAttemptId ); Org.Mockito.Mockito.When(mockLaunchEvent.GetContainerMgrAddress()).ThenReturn(cmAddress ); Org.Mockito.Mockito.When(mockLaunchEvent.GetContainerToken()).ThenReturn(CreateNewContainerToken (contId, cmAddress)); ut.Handle(mockLaunchEvent); startLaunchBarrier.Await(); Log.Info("inserting cleanup event"); ContainerLauncherEvent mockCleanupEvent = Org.Mockito.Mockito.Mock <ContainerLauncherEvent >(); Org.Mockito.Mockito.When(mockCleanupEvent.GetType()).ThenReturn(ContainerLauncher.EventType .ContainerRemoteCleanup); Org.Mockito.Mockito.When(mockCleanupEvent.GetContainerID()).ThenReturn(contId); Org.Mockito.Mockito.When(mockCleanupEvent.GetTaskAttemptID()).ThenReturn(taskAttemptId ); Org.Mockito.Mockito.When(mockCleanupEvent.GetContainerMgrAddress()).ThenReturn(cmAddress ); ut.Handle(mockCleanupEvent); completeLaunchBarrier.Await(); ut.WaitForPoolToIdle(); ArgumentCaptor <Org.Apache.Hadoop.Yarn.Event.Event> arg = ArgumentCaptor.ForClass < Org.Apache.Hadoop.Yarn.Event.Event>(); Org.Mockito.Mockito.Verify(mockEventHandler, Org.Mockito.Mockito.AtLeast(2)).Handle (arg.Capture()); bool containerCleaned = false; for (int i = 0; i < arg.GetAllValues().Count; i++) { Log.Info(arg.GetAllValues()[i].ToString()); Org.Apache.Hadoop.Yarn.Event.Event currentEvent = arg.GetAllValues()[i]; if (currentEvent.GetType() == TaskAttemptEventType.TaContainerCleaned) { containerCleaned = true; } } System.Diagnostics.Debug.Assert((containerCleaned)); } finally { ut.Stop(); } }