protected override void Dispatch(Org.Apache.Hadoop.Yarn.Event.Event @event)
{
if (@event is TaskAttemptEvent)
{
TaskAttemptEvent killEvent = (TaskAttemptEvent)@event;
if (killEvent.GetType() == TaskAttemptEventType.TaKill)
{
TaskAttemptId taID = killEvent.GetTaskAttemptID();
if (taID.GetTaskId().GetTaskType() == TaskType.Reduce && taID.GetTaskId().GetId()
== 0 && taID.GetId() == 0)
{
base.Dispatch(new TaskAttemptEvent(taID, TaskAttemptEventType.TaDone));
base.Dispatch(new TaskAttemptEvent(taID, TaskAttemptEventType.TaContainerCleaned)
);
base.Dispatch(new TaskTAttemptEvent(taID, TaskEventType.TAttemptSucceeded));
this.cachedKillEvent = killEvent;
return;
}
}
}
else
{
if (@event is TaskEvent)
{
TaskEvent taskEvent = (TaskEvent)@event;
if (taskEvent.GetType() == TaskEventType.TAttemptSucceeded && this.cachedKillEvent
!= null)
{
base.Dispatch(this.cachedKillEvent);
return;
}
}
}
base.Dispatch(@event);
}
public virtual void Handle(Org.Apache.Hadoop.Yarn.Event.Event @event)
{
lock (this)
{
this.@event = @event;
Sharpen.Runtime.NotifyAll(this);
}
}
protected override void Dispatch(Org.Apache.Hadoop.Yarn.Event.Event @event)
{
if (@event.GetType() == RMFatalEventType.TransitionToActiveFailed)
{
this.eventreceived++;
}
else
{
base.Dispatch(@event);
}
}
public void Handle(Org.Apache.Hadoop.Yarn.Event.Event @event)
{
TestLocalContainerLauncher.Log.Info("handling event " + @event.GetType() + " with type "
+ @event.GetType());
if (@event is TaskAttemptEvent)
{
if (@event.GetType() == TaskAttemptEventType.TaContainerCleaned)
{
isDone.CountDown();
}
}
}
public virtual void TestBasic()
{
AppContext mockContext = Org.Mockito.Mockito.Mock <AppContext>();
OutputCommitter mockCommitter = Org.Mockito.Mockito.Mock <OutputCommitter>();
Clock mockClock = Org.Mockito.Mockito.Mock <Clock>();
CommitterEventHandler handler = new CommitterEventHandler(mockContext, mockCommitter
, new TestCommitterEventHandler.TestingRMHeartbeatHandler());
YarnConfiguration conf = new YarnConfiguration();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
JobContext mockJobContext = Org.Mockito.Mockito.Mock <JobContext>();
ApplicationAttemptId attemptid = ConverterUtils.ToApplicationAttemptId("appattempt_1234567890000_0001_0"
);
JobId jobId = TypeConverter.ToYarn(TypeConverter.FromYarn(attemptid.GetApplicationId
()));
TestCommitterEventHandler.WaitForItHandler waitForItHandler = new TestCommitterEventHandler.WaitForItHandler
();
Org.Mockito.Mockito.When(mockContext.GetApplicationID()).ThenReturn(attemptid.GetApplicationId
());
Org.Mockito.Mockito.When(mockContext.GetApplicationAttemptId()).ThenReturn(attemptid
);
Org.Mockito.Mockito.When(mockContext.GetEventHandler()).ThenReturn(waitForItHandler
);
Org.Mockito.Mockito.When(mockContext.GetClock()).ThenReturn(mockClock);
handler.Init(conf);
handler.Start();
try
{
handler.Handle(new CommitterJobCommitEvent(jobId, mockJobContext));
string user = UserGroupInformation.GetCurrentUser().GetShortUserName();
Path startCommitFile = MRApps.GetStartJobCommitFile(conf, user, jobId);
Path endCommitSuccessFile = MRApps.GetEndJobCommitSuccessFile(conf, user, jobId);
Path endCommitFailureFile = MRApps.GetEndJobCommitFailureFile(conf, user, jobId);
Org.Apache.Hadoop.Yarn.Event.Event e = waitForItHandler.GetAndClearEvent();
NUnit.Framework.Assert.IsNotNull(e);
NUnit.Framework.Assert.IsTrue(e is JobCommitCompletedEvent);
FileSystem fs = FileSystem.Get(conf);
NUnit.Framework.Assert.IsTrue(startCommitFile.ToString(), fs.Exists(startCommitFile
));
NUnit.Framework.Assert.IsTrue(endCommitSuccessFile.ToString(), fs.Exists(endCommitSuccessFile
));
NUnit.Framework.Assert.IsFalse(endCommitFailureFile.ToString(), fs.Exists(endCommitFailureFile
));
Org.Mockito.Mockito.Verify(mockCommitter).CommitJob(Matchers.Any <JobContext>());
}
finally
{
handler.Stop();
}
}
protected override void Dispatch(Org.Apache.Hadoop.Yarn.Event.Event @event)
{
if (@event is WritingApplicationHistoryEvent)
{
WritingApplicationHistoryEvent ashEvent = (WritingApplicationHistoryEvent)@event;
switch (ashEvent.GetType())
{
case WritingHistoryEventType.AppStart:
{
IncrementCounts(((WritingApplicationStartEvent)@event).GetApplicationId());
break;
}
case WritingHistoryEventType.AppFinish:
{
IncrementCounts(((WritingApplicationFinishEvent)@event).GetApplicationId());
break;
}
case WritingHistoryEventType.AppAttemptStart:
{
IncrementCounts(((WritingApplicationAttemptStartEvent)@event).GetApplicationAttemptId
().GetApplicationId());
break;
}
case WritingHistoryEventType.AppAttemptFinish:
{
IncrementCounts(((WritingApplicationAttemptFinishEvent)@event).GetApplicationAttemptId
().GetApplicationId());
break;
}
case WritingHistoryEventType.ContainerStart:
{
IncrementCounts(((WritingContainerStartEvent)@event).GetContainerId().GetApplicationAttemptId
().GetApplicationId());
break;
}
case WritingHistoryEventType.ContainerFinish:
{
IncrementCounts(((WritingContainerFinishEvent)@event).GetContainerId().GetApplicationAttemptId
().GetApplicationId());
break;
}
}
}
base.Dispatch(@event);
}
/// <exception cref="System.Exception"/>
public virtual Org.Apache.Hadoop.Yarn.Event.Event GetAndClearEvent()
{
lock (this)
{
if (@event == null)
{
long waitTime = 5000;
long waitStartTime = Time.MonotonicNow();
while (@event == null && Time.MonotonicNow() - waitStartTime < waitTime)
{
//Wait for at most 5 sec
Sharpen.Runtime.Wait(this, waitTime);
}
}
Org.Apache.Hadoop.Yarn.Event.Event e = @event;
@event = null;
return(e);
}
}
protected override void Dispatch(Org.Apache.Hadoop.Yarn.Event.Event @event)
{
if (@event is TaskAttemptEvent)
{
TaskAttemptEvent attemptEvent = (TaskAttemptEvent)@event;
TaskAttemptId attemptID = ((TaskAttemptEvent)@event).GetTaskAttemptID();
if (attemptEvent.GetType() == this.attemptEventTypeToWait && attemptID.GetTaskId(
).GetId() == 0 && attemptID.GetId() == 0)
{
try
{
latch.Await();
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
}
}
base.Dispatch(@event);
}
public void Handle(Org.Apache.Hadoop.Yarn.Event.Event @event)
{
}
/// <exception cref="System.Exception"/>
public virtual void TestContainerCleaned()
{
Log.Info("STARTING testContainerCleaned");
CyclicBarrier startLaunchBarrier = new CyclicBarrier(2);
CyclicBarrier completeLaunchBarrier = new CyclicBarrier(2);
AppContext mockContext = Org.Mockito.Mockito.Mock <AppContext>();
EventHandler mockEventHandler = Org.Mockito.Mockito.Mock <EventHandler>();
Org.Mockito.Mockito.When(mockContext.GetEventHandler()).ThenReturn(mockEventHandler
);
TestContainerLauncherImpl.ContainerManagementProtocolClient mockCM = new TestContainerLauncherImpl.ContainerManagerForTest
(startLaunchBarrier, completeLaunchBarrier);
TestContainerLauncherImpl.ContainerLauncherImplUnderTest ut = new TestContainerLauncherImpl.ContainerLauncherImplUnderTest
(mockContext, mockCM);
Configuration conf = new Configuration();
ut.Init(conf);
ut.Start();
try
{
ContainerId contId = MakeContainerId(0l, 0, 0, 1);
TaskAttemptId taskAttemptId = MakeTaskAttemptId(0l, 0, 0, TaskType.Map, 0);
string cmAddress = "127.0.0.1:8000";
StartContainersResponse startResp = recordFactory.NewRecordInstance <StartContainersResponse
>();
startResp.SetAllServicesMetaData(serviceResponse);
Log.Info("inserting launch event");
ContainerRemoteLaunchEvent mockLaunchEvent = Org.Mockito.Mockito.Mock <ContainerRemoteLaunchEvent
>();
Org.Mockito.Mockito.When(mockLaunchEvent.GetType()).ThenReturn(ContainerLauncher.EventType
.ContainerRemoteLaunch);
Org.Mockito.Mockito.When(mockLaunchEvent.GetContainerID()).ThenReturn(contId);
Org.Mockito.Mockito.When(mockLaunchEvent.GetTaskAttemptID()).ThenReturn(taskAttemptId
);
Org.Mockito.Mockito.When(mockLaunchEvent.GetContainerMgrAddress()).ThenReturn(cmAddress
);
Org.Mockito.Mockito.When(mockLaunchEvent.GetContainerToken()).ThenReturn(CreateNewContainerToken
(contId, cmAddress));
ut.Handle(mockLaunchEvent);
startLaunchBarrier.Await();
Log.Info("inserting cleanup event");
ContainerLauncherEvent mockCleanupEvent = Org.Mockito.Mockito.Mock <ContainerLauncherEvent
>();
Org.Mockito.Mockito.When(mockCleanupEvent.GetType()).ThenReturn(ContainerLauncher.EventType
.ContainerRemoteCleanup);
Org.Mockito.Mockito.When(mockCleanupEvent.GetContainerID()).ThenReturn(contId);
Org.Mockito.Mockito.When(mockCleanupEvent.GetTaskAttemptID()).ThenReturn(taskAttemptId
);
Org.Mockito.Mockito.When(mockCleanupEvent.GetContainerMgrAddress()).ThenReturn(cmAddress
);
ut.Handle(mockCleanupEvent);
completeLaunchBarrier.Await();
ut.WaitForPoolToIdle();
ArgumentCaptor <Org.Apache.Hadoop.Yarn.Event.Event> arg = ArgumentCaptor.ForClass <
Org.Apache.Hadoop.Yarn.Event.Event>();
Org.Mockito.Mockito.Verify(mockEventHandler, Org.Mockito.Mockito.AtLeast(2)).Handle
(arg.Capture());
bool containerCleaned = false;
for (int i = 0; i < arg.GetAllValues().Count; i++)
{
Log.Info(arg.GetAllValues()[i].ToString());
Org.Apache.Hadoop.Yarn.Event.Event currentEvent = arg.GetAllValues()[i];
if (currentEvent.GetType() == TaskAttemptEventType.TaContainerCleaned)
{
containerCleaned = true;
}
}
System.Diagnostics.Debug.Assert((containerCleaned));
}
finally
{
ut.Stop();
}
}