public RMContainerState Transition(RMContainerImpl container, RMContainerEvent @event
)
{
NMContainerStatus report = ((RMContainerRecoverEvent)@event).GetContainerReport();
if (report.GetContainerState().Equals(ContainerState.Complete))
{
ContainerStatus status = ContainerStatus.NewInstance(report.GetContainerId(), report
.GetContainerState(), report.GetDiagnostics(), report.GetContainerExitStatus());
new RMContainerImpl.FinishedTransition().Transition(container, new RMContainerFinishedEvent
(container.containerId, status, RMContainerEventType.Finished));
return(RMContainerState.Completed);
}
else
{
if (report.GetContainerState().Equals(ContainerState.Running))
{
// Tell the app
container.eventHandler.Handle(new RMAppRunningOnNodeEvent(container.GetApplicationAttemptId
().GetApplicationId(), container.nodeId));
return(RMContainerState.Running);
}
else
{
// This can never happen.
Log.Warn("RMContainer received unexpected recover event with container" + " state "
+ report.GetContainerState() + " while recovering.");
return(RMContainerState.Running);
}
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestCallAMRMClientAsyncStopFromCallbackHandlerWithWaitFor()
{
Configuration conf = new Configuration();
TestAMRMClientAsync.TestCallbackHandler2 callbackHandler = new TestAMRMClientAsync.TestCallbackHandler2
(this);
AMRMClient <AMRMClient.ContainerRequest> client = Org.Mockito.Mockito.Mock <AMRMClientImpl
>();
IList <ContainerStatus> completed = Arrays.AsList(ContainerStatus.NewInstance(NewContainerId
(0, 0, 0, 0), ContainerState.Complete, string.Empty, 0));
AllocateResponse response = CreateAllocateResponse(completed, new AList <Container
>(), null);
Org.Mockito.Mockito.When(client.Allocate(Matchers.AnyFloat())).ThenReturn(response
);
AMRMClientAsync <AMRMClient.ContainerRequest> asyncClient = AMRMClientAsync.CreateAMRMClientAsync
(client, 20, callbackHandler);
callbackHandler.asynClient = asyncClient;
asyncClient.Init(conf);
asyncClient.Start();
Supplier <bool> checker = new _Supplier_320(callbackHandler);
asyncClient.RegisterApplicationMaster("localhost", 1234, null);
asyncClient.WaitFor(checker);
NUnit.Framework.Assert.IsTrue(checker.Get());
}
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
internal virtual void RunCallBackThrowOutException(TestAMRMClientAsync.TestCallbackHandler2
callbackHandler)
{
Configuration conf = new Configuration();
AMRMClient <AMRMClient.ContainerRequest> client = Org.Mockito.Mockito.Mock <AMRMClientImpl
>();
IList <ContainerStatus> completed = Arrays.AsList(ContainerStatus.NewInstance(NewContainerId
(0, 0, 0, 0), ContainerState.Complete, string.Empty, 0));
AllocateResponse response = CreateAllocateResponse(completed, new AList <Container
>(), null);
Org.Mockito.Mockito.When(client.Allocate(Matchers.AnyFloat())).ThenReturn(response
);
AMRMClientAsync <AMRMClient.ContainerRequest> asyncClient = AMRMClientAsync.CreateAMRMClientAsync
(client, 20, callbackHandler);
callbackHandler.asynClient = asyncClient;
callbackHandler.throwOutException = true;
asyncClient.Init(conf);
asyncClient.Start();
// call register and wait for error callback and stop
lock (callbackHandler.notifier)
{
asyncClient.RegisterApplicationMaster("localhost", 1234, null);
while (callbackHandler.notify == false)
{
try
{
Sharpen.Runtime.Wait(callbackHandler.notifier);
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
}
}
// verify error invoked
Org.Mockito.Mockito.Verify(callbackHandler, Org.Mockito.Mockito.Times(0)).GetProgress
();
Org.Mockito.Mockito.Verify(callbackHandler, Org.Mockito.Mockito.Times(1)).OnError
(Matchers.Any <Exception>());
// sleep to wait for a few heartbeat calls that can trigger callbacks
Sharpen.Thread.Sleep(50);
// verify no more invocations after the first one.
// ie. callback thread has stopped
Org.Mockito.Mockito.Verify(callbackHandler, Org.Mockito.Mockito.Times(0)).GetProgress
();
Org.Mockito.Mockito.Verify(callbackHandler, Org.Mockito.Mockito.Times(1)).OnError
(Matchers.Any <Exception>());
}
internal virtual void HandleNMContainerStatus(NMContainerStatus containerStatus,
NodeId nodeId)
{
ApplicationAttemptId appAttemptId = containerStatus.GetContainerId().GetApplicationAttemptId
();
RMApp rmApp = rmContext.GetRMApps()[appAttemptId.GetApplicationId()];
if (rmApp == null)
{
Log.Error("Received finished container : " + containerStatus.GetContainerId() + " for unknown application "
+ appAttemptId.GetApplicationId() + " Skipping.");
return;
}
if (rmApp.GetApplicationSubmissionContext().GetUnmanagedAM())
{
if (Log.IsDebugEnabled())
{
Log.Debug("Ignoring container completion status for unmanaged AM " + rmApp.GetApplicationId
());
}
return;
}
RMAppAttempt rmAppAttempt = rmApp.GetRMAppAttempt(appAttemptId);
Container masterContainer = rmAppAttempt.GetMasterContainer();
if (masterContainer.GetId().Equals(containerStatus.GetContainerId()) && containerStatus
.GetContainerState() == ContainerState.Complete)
{
ContainerStatus status = ContainerStatus.NewInstance(containerStatus.GetContainerId
(), containerStatus.GetContainerState(), containerStatus.GetDiagnostics(), containerStatus
.GetContainerExitStatus());
// sending master container finished event.
RMAppAttemptContainerFinishedEvent evt = new RMAppAttemptContainerFinishedEvent(appAttemptId
, status, nodeId);
rmContext.GetDispatcher().GetEventHandler().Handle(evt);
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestCallAMRMClientAsyncStopFromCallbackHandler()
{
Configuration conf = new Configuration();
TestAMRMClientAsync.TestCallbackHandler2 callbackHandler = new TestAMRMClientAsync.TestCallbackHandler2
(this);
AMRMClient <AMRMClient.ContainerRequest> client = Org.Mockito.Mockito.Mock <AMRMClientImpl
>();
IList <ContainerStatus> completed = Arrays.AsList(ContainerStatus.NewInstance(NewContainerId
(0, 0, 0, 0), ContainerState.Complete, string.Empty, 0));
AllocateResponse response = CreateAllocateResponse(completed, new AList <Container
>(), null);
Org.Mockito.Mockito.When(client.Allocate(Matchers.AnyFloat())).ThenReturn(response
);
AMRMClientAsync <AMRMClient.ContainerRequest> asyncClient = AMRMClientAsync.CreateAMRMClientAsync
(client, 20, callbackHandler);
callbackHandler.asynClient = asyncClient;
asyncClient.Init(conf);
asyncClient.Start();
lock (callbackHandler.notifier)
{
asyncClient.RegisterApplicationMaster("localhost", 1234, null);
while (callbackHandler.notify == false)
{
try
{
Sharpen.Runtime.Wait(callbackHandler.notifier);
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestAMRMClientAsync()
{
Configuration conf = new Configuration();
AtomicBoolean heartbeatBlock = new AtomicBoolean(true);
IList <ContainerStatus> completed1 = Arrays.AsList(ContainerStatus.NewInstance(NewContainerId
(0, 0, 0, 0), ContainerState.Complete, string.Empty, 0));
IList <Container> allocated1 = Arrays.AsList(Container.NewInstance(null, null, null
, null, null, null));
AllocateResponse response1 = CreateAllocateResponse(new AList <ContainerStatus>(),
allocated1, null);
AllocateResponse response2 = CreateAllocateResponse(completed1, new AList <Container
>(), null);
AllocateResponse emptyResponse = CreateAllocateResponse(new AList <ContainerStatus
>(), new AList <Container>(), null);
TestAMRMClientAsync.TestCallbackHandler callbackHandler = new TestAMRMClientAsync.TestCallbackHandler
(this);
AMRMClient <AMRMClient.ContainerRequest> client = Org.Mockito.Mockito.Mock <AMRMClientImpl
>();
AtomicInteger secondHeartbeatSync = new AtomicInteger(0);
Org.Mockito.Mockito.When(client.Allocate(Matchers.AnyFloat())).ThenReturn(response1
).ThenAnswer(new _Answer_89(secondHeartbeatSync, heartbeatBlock, response2)).ThenReturn
(emptyResponse);
Org.Mockito.Mockito.When(client.RegisterApplicationMaster(Matchers.AnyString(), Matchers.AnyInt
(), Matchers.AnyString())).ThenReturn(null);
Org.Mockito.Mockito.When(client.GetAvailableResources()).ThenAnswer(new _Answer_105
(client));
// take client lock to simulate behavior of real impl
AMRMClientAsync <AMRMClient.ContainerRequest> asyncClient = AMRMClientAsync.CreateAMRMClientAsync
(client, 20, callbackHandler);
asyncClient.Init(conf);
asyncClient.Start();
asyncClient.RegisterApplicationMaster("localhost", 1234, null);
// while the CallbackHandler will still only be processing the first response,
// heartbeater thread should still be sending heartbeats.
// To test this, wait for the second heartbeat to be received.
while (secondHeartbeatSync.Get() < 1)
{
Sharpen.Thread.Sleep(10);
}
// heartbeat will be blocked. make sure we can call client methods at this
// time. Checks that heartbeat is not holding onto client lock
System.Diagnostics.Debug.Assert((secondHeartbeatSync.Get() < 2));
asyncClient.GetAvailableResources();
// method returned. now unblock heartbeat
System.Diagnostics.Debug.Assert((secondHeartbeatSync.Get() < 2));
lock (heartbeatBlock)
{
heartbeatBlock.Set(false);
Sharpen.Runtime.NotifyAll(heartbeatBlock);
}
// allocated containers should come before completed containers
NUnit.Framework.Assert.AreEqual(null, callbackHandler.TakeCompletedContainers());
// wait for the allocated containers from the first heartbeat's response
while (callbackHandler.TakeAllocatedContainers() == null)
{
NUnit.Framework.Assert.AreEqual(null, callbackHandler.TakeCompletedContainers());
Sharpen.Thread.Sleep(10);
}
// wait for the completed containers from the second heartbeat's response
while (callbackHandler.TakeCompletedContainers() == null)
{
Sharpen.Thread.Sleep(10);
}
asyncClient.Stop();
NUnit.Framework.Assert.AreEqual(null, callbackHandler.TakeAllocatedContainers());
NUnit.Framework.Assert.AreEqual(null, callbackHandler.TakeCompletedContainers());
}