private CustomEventProcessor(CustomContainerLauncher _enclosing, ContainerLauncherEvent
@event)
: base(_enclosing)
{
this._enclosing = _enclosing;
this.@event = @event;
}
public virtual void Handle(ContainerLauncherEvent @event)
{
try
{
eventQueue.Put(@event);
}
catch (Exception e)
{
throw new YarnRuntimeException(e);
}
}
protected internal override ContainerLauncherImpl.EventProcessor CreateEventProcessor
(ContainerLauncherEvent @event)
{
// At this point of time, the EventProcessor is being created and so no
// additional threads would have been created.
// Core-pool-size should have increased by now.
if (this.expectedCorePoolSize != this.launcherPool.GetCorePoolSize())
{
this.foundErrors = "Expected " + this.expectedCorePoolSize + " but found " + this
.launcherPool.GetCorePoolSize();
}
return new TestContainerLauncher.CustomContainerLauncher.CustomEventProcessor(this
, @event);
}
private ContainerLauncherImpl.Container GetContainer(ContainerLauncherEvent @event
)
{
ContainerId id = @event.GetContainerID();
ContainerLauncherImpl.Container c = containers[id];
if (c == null)
{
c = new ContainerLauncherImpl.Container(this, @event.GetTaskAttemptID(), @event.GetContainerID
(), @event.GetContainerMgrAddress());
ContainerLauncherImpl.Container old = containers.PutIfAbsent(id, c);
if (old != null)
{
c = old;
}
}
return(c);
}
public override void Run()
{
ContainerLauncherEvent @event = null;
ICollection <string> allNodes = new HashSet <string>();
while (!this._enclosing.stopped.Get() && !Sharpen.Thread.CurrentThread().IsInterrupted
())
{
try
{
@event = this._enclosing.eventQueue.Take();
}
catch (Exception e)
{
if (!this._enclosing.stopped.Get())
{
ContainerLauncherImpl.Log.Error("Returning, interrupted : " + e);
}
return;
}
allNodes.AddItem(@event.GetContainerMgrAddress());
int poolSize = this._enclosing.launcherPool.GetCorePoolSize();
if (poolSize != this._enclosing.limitOnPoolSize)
{
int numNodes = allNodes.Count;
int idealPoolSize = Math.Min(this._enclosing.limitOnPoolSize, numNodes);
if (poolSize < idealPoolSize)
{
int newPoolSize = Math.Min(this._enclosing.limitOnPoolSize, idealPoolSize + this.
_enclosing.initialPoolSize);
ContainerLauncherImpl.Log.Info("Setting ContainerLauncher pool size to " + newPoolSize
+ " as number-of-nodes to talk to is " + numNodes);
this._enclosing.launcherPool.SetCorePoolSize(newPoolSize);
}
}
this._enclosing.launcherPool.Execute(this._enclosing.CreateEventProcessor(@event)
);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestContainerCleaned()
{
Log.Info("STARTING testContainerCleaned");
CyclicBarrier startLaunchBarrier = new CyclicBarrier(2);
CyclicBarrier completeLaunchBarrier = new CyclicBarrier(2);
AppContext mockContext = Org.Mockito.Mockito.Mock <AppContext>();
EventHandler mockEventHandler = Org.Mockito.Mockito.Mock <EventHandler>();
Org.Mockito.Mockito.When(mockContext.GetEventHandler()).ThenReturn(mockEventHandler
);
TestContainerLauncherImpl.ContainerManagementProtocolClient mockCM = new TestContainerLauncherImpl.ContainerManagerForTest
(startLaunchBarrier, completeLaunchBarrier);
TestContainerLauncherImpl.ContainerLauncherImplUnderTest ut = new TestContainerLauncherImpl.ContainerLauncherImplUnderTest
(mockContext, mockCM);
Configuration conf = new Configuration();
ut.Init(conf);
ut.Start();
try
{
ContainerId contId = MakeContainerId(0l, 0, 0, 1);
TaskAttemptId taskAttemptId = MakeTaskAttemptId(0l, 0, 0, TaskType.Map, 0);
string cmAddress = "127.0.0.1:8000";
StartContainersResponse startResp = recordFactory.NewRecordInstance <StartContainersResponse
>();
startResp.SetAllServicesMetaData(serviceResponse);
Log.Info("inserting launch event");
ContainerRemoteLaunchEvent mockLaunchEvent = Org.Mockito.Mockito.Mock <ContainerRemoteLaunchEvent
>();
Org.Mockito.Mockito.When(mockLaunchEvent.GetType()).ThenReturn(ContainerLauncher.EventType
.ContainerRemoteLaunch);
Org.Mockito.Mockito.When(mockLaunchEvent.GetContainerID()).ThenReturn(contId);
Org.Mockito.Mockito.When(mockLaunchEvent.GetTaskAttemptID()).ThenReturn(taskAttemptId
);
Org.Mockito.Mockito.When(mockLaunchEvent.GetContainerMgrAddress()).ThenReturn(cmAddress
);
Org.Mockito.Mockito.When(mockLaunchEvent.GetContainerToken()).ThenReturn(CreateNewContainerToken
(contId, cmAddress));
ut.Handle(mockLaunchEvent);
startLaunchBarrier.Await();
Log.Info("inserting cleanup event");
ContainerLauncherEvent mockCleanupEvent = Org.Mockito.Mockito.Mock <ContainerLauncherEvent
>();
Org.Mockito.Mockito.When(mockCleanupEvent.GetType()).ThenReturn(ContainerLauncher.EventType
.ContainerRemoteCleanup);
Org.Mockito.Mockito.When(mockCleanupEvent.GetContainerID()).ThenReturn(contId);
Org.Mockito.Mockito.When(mockCleanupEvent.GetTaskAttemptID()).ThenReturn(taskAttemptId
);
Org.Mockito.Mockito.When(mockCleanupEvent.GetContainerMgrAddress()).ThenReturn(cmAddress
);
ut.Handle(mockCleanupEvent);
completeLaunchBarrier.Await();
ut.WaitForPoolToIdle();
ArgumentCaptor <Org.Apache.Hadoop.Yarn.Event.Event> arg = ArgumentCaptor.ForClass <
Org.Apache.Hadoop.Yarn.Event.Event>();
Org.Mockito.Mockito.Verify(mockEventHandler, Org.Mockito.Mockito.AtLeast(2)).Handle
(arg.Capture());
bool containerCleaned = false;
for (int i = 0; i < arg.GetAllValues().Count; i++)
{
Log.Info(arg.GetAllValues()[i].ToString());
Org.Apache.Hadoop.Yarn.Event.Event currentEvent = arg.GetAllValues()[i];
if (currentEvent.GetType() == TaskAttemptEventType.TaContainerCleaned)
{
containerCleaned = true;
}
}
System.Diagnostics.Debug.Assert((containerCleaned));
}
finally
{
ut.Stop();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestOutOfOrder()
{
Log.Info("STARTING testOutOfOrder");
AppContext mockContext = Org.Mockito.Mockito.Mock <AppContext>();
EventHandler mockEventHandler = Org.Mockito.Mockito.Mock <EventHandler>();
Org.Mockito.Mockito.When(mockContext.GetEventHandler()).ThenReturn(mockEventHandler
);
TestContainerLauncherImpl.ContainerManagementProtocolClient mockCM = Org.Mockito.Mockito.Mock
<TestContainerLauncherImpl.ContainerManagementProtocolClient>();
TestContainerLauncherImpl.ContainerLauncherImplUnderTest ut = new TestContainerLauncherImpl.ContainerLauncherImplUnderTest
(mockContext, mockCM);
Configuration conf = new Configuration();
ut.Init(conf);
ut.Start();
try
{
ContainerId contId = MakeContainerId(0l, 0, 0, 1);
TaskAttemptId taskAttemptId = MakeTaskAttemptId(0l, 0, 0, TaskType.Map, 0);
string cmAddress = "127.0.0.1:8000";
StartContainersResponse startResp = recordFactory.NewRecordInstance <StartContainersResponse
>();
startResp.SetAllServicesMetaData(serviceResponse);
Log.Info("inserting cleanup event");
ContainerLauncherEvent mockCleanupEvent = Org.Mockito.Mockito.Mock <ContainerLauncherEvent
>();
Org.Mockito.Mockito.When(mockCleanupEvent.GetType()).ThenReturn(ContainerLauncher.EventType
.ContainerRemoteCleanup);
Org.Mockito.Mockito.When(mockCleanupEvent.GetContainerID()).ThenReturn(contId);
Org.Mockito.Mockito.When(mockCleanupEvent.GetTaskAttemptID()).ThenReturn(taskAttemptId
);
Org.Mockito.Mockito.When(mockCleanupEvent.GetContainerMgrAddress()).ThenReturn(cmAddress
);
ut.Handle(mockCleanupEvent);
ut.WaitForPoolToIdle();
Org.Mockito.Mockito.Verify(mockCM, Org.Mockito.Mockito.Never()).StopContainers(Matchers.Any
<StopContainersRequest>());
Log.Info("inserting launch event");
ContainerRemoteLaunchEvent mockLaunchEvent = Org.Mockito.Mockito.Mock <ContainerRemoteLaunchEvent
>();
Org.Mockito.Mockito.When(mockLaunchEvent.GetType()).ThenReturn(ContainerLauncher.EventType
.ContainerRemoteLaunch);
Org.Mockito.Mockito.When(mockLaunchEvent.GetContainerID()).ThenReturn(contId);
Org.Mockito.Mockito.When(mockLaunchEvent.GetTaskAttemptID()).ThenReturn(taskAttemptId
);
Org.Mockito.Mockito.When(mockLaunchEvent.GetContainerMgrAddress()).ThenReturn(cmAddress
);
Org.Mockito.Mockito.When(mockCM.StartContainers(Matchers.Any <StartContainersRequest
>())).ThenReturn(startResp);
Org.Mockito.Mockito.When(mockLaunchEvent.GetContainerToken()).ThenReturn(CreateNewContainerToken
(contId, cmAddress));
ut.Handle(mockLaunchEvent);
ut.WaitForPoolToIdle();
Org.Mockito.Mockito.Verify(mockCM, Org.Mockito.Mockito.Never()).StartContainers(Matchers.Any
<StartContainersRequest>());
}
finally
{
ut.Stop();
}
}
internal EventProcessor(ContainerLauncherImpl _enclosing, ContainerLauncherEvent
@event)
{
this._enclosing = _enclosing;
this.@event = @event;
}
protected internal virtual ContainerLauncherImpl.EventProcessor CreateEventProcessor
(ContainerLauncherEvent @event)
{
return(new ContainerLauncherImpl.EventProcessor(this, @event));
}
