/// <exception cref="System.Exception"/>
public virtual void TestAMRMClientAsyncShutDownWithWaitFor()
{
Configuration conf = new Configuration();
TestAMRMClientAsync.TestCallbackHandler callbackHandler = new TestAMRMClientAsync.TestCallbackHandler
(this);
AMRMClient <AMRMClient.ContainerRequest> client = Org.Mockito.Mockito.Mock <AMRMClientImpl
>();
Org.Mockito.Mockito.When(client.Allocate(Matchers.AnyFloat())).ThenThrow(new ApplicationAttemptNotFoundException
("app not found, shut down"));
AMRMClientAsync <AMRMClient.ContainerRequest> asyncClient = AMRMClientAsync.CreateAMRMClientAsync
(client, 10, callbackHandler);
asyncClient.Init(conf);
asyncClient.Start();
Supplier <bool> checker = new _Supplier_246(callbackHandler);
asyncClient.RegisterApplicationMaster("localhost", 1234, null);
asyncClient.WaitFor(checker);
asyncClient.Stop();
// stopping should have joined all threads and completed all callbacks
NUnit.Framework.Assert.IsTrue(callbackHandler.callbackCount == 0);
Org.Mockito.Mockito.Verify(client, Org.Mockito.Mockito.Times(1)).Allocate(Matchers.AnyFloat
());
asyncClient.Stop();
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestCallAMRMClientAsyncStopFromCallbackHandlerWithWaitFor()
{
Configuration conf = new Configuration();
TestAMRMClientAsync.TestCallbackHandler2 callbackHandler = new TestAMRMClientAsync.TestCallbackHandler2
(this);
AMRMClient <AMRMClient.ContainerRequest> client = Org.Mockito.Mockito.Mock <AMRMClientImpl
>();
IList <ContainerStatus> completed = Arrays.AsList(ContainerStatus.NewInstance(NewContainerId
(0, 0, 0, 0), ContainerState.Complete, string.Empty, 0));
AllocateResponse response = CreateAllocateResponse(completed, new AList <Container
>(), null);
Org.Mockito.Mockito.When(client.Allocate(Matchers.AnyFloat())).ThenReturn(response
);
AMRMClientAsync <AMRMClient.ContainerRequest> asyncClient = AMRMClientAsync.CreateAMRMClientAsync
(client, 20, callbackHandler);
callbackHandler.asynClient = asyncClient;
asyncClient.Init(conf);
asyncClient.Start();
Supplier <bool> checker = new _Supplier_320(callbackHandler);
asyncClient.RegisterApplicationMaster("localhost", 1234, null);
asyncClient.WaitFor(checker);
NUnit.Framework.Assert.IsTrue(checker.Get());
}
public static Org.Apache.Hadoop.Yarn.Client.Api.Async.AMRMClientAsync <T> CreateAMRMClientAsync
<T>(AMRMClient <T> client, int intervalMs, AMRMClientAsync.CallbackHandler callbackHandler
)
where T : AMRMClient.ContainerRequest
{
return(new AMRMClientAsyncImpl <T>(client, intervalMs, callbackHandler));
}
protected internal AMRMClientAsync(AMRMClient <T> client, int intervalMs, AMRMClientAsync.CallbackHandler
callbackHandler)
: base(typeof(Org.Apache.Hadoop.Yarn.Client.Api.Async.AMRMClientAsync).FullName)
{
this.client = client;
this.heartbeatIntervalMs.Set(intervalMs);
this.handler = callbackHandler;
}
public AMRMClientAsyncImpl(AMRMClient <T> client, int intervalMs, AMRMClientAsync.CallbackHandler
callbackHandler)
: base(client, intervalMs, callbackHandler)
{
heartbeatThread = new AMRMClientAsyncImpl.HeartbeatThread(this);
handlerThread = new AMRMClientAsyncImpl.CallbackHandlerThread(this);
responseQueue = new LinkedBlockingQueue <AllocateResponse>();
keepRunning = true;
savedException = null;
}
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
internal virtual void RunCallBackThrowOutException(TestAMRMClientAsync.TestCallbackHandler2
callbackHandler)
{
Configuration conf = new Configuration();
AMRMClient <AMRMClient.ContainerRequest> client = Org.Mockito.Mockito.Mock <AMRMClientImpl
>();
IList <ContainerStatus> completed = Arrays.AsList(ContainerStatus.NewInstance(NewContainerId
(0, 0, 0, 0), ContainerState.Complete, string.Empty, 0));
AllocateResponse response = CreateAllocateResponse(completed, new AList <Container
>(), null);
Org.Mockito.Mockito.When(client.Allocate(Matchers.AnyFloat())).ThenReturn(response
);
AMRMClientAsync <AMRMClient.ContainerRequest> asyncClient = AMRMClientAsync.CreateAMRMClientAsync
(client, 20, callbackHandler);
callbackHandler.asynClient = asyncClient;
callbackHandler.throwOutException = true;
asyncClient.Init(conf);
asyncClient.Start();
// call register and wait for error callback and stop
lock (callbackHandler.notifier)
{
asyncClient.RegisterApplicationMaster("localhost", 1234, null);
while (callbackHandler.notify == false)
{
try
{
Sharpen.Runtime.Wait(callbackHandler.notifier);
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
}
}
// verify error invoked
Org.Mockito.Mockito.Verify(callbackHandler, Org.Mockito.Mockito.Times(0)).GetProgress
();
Org.Mockito.Mockito.Verify(callbackHandler, Org.Mockito.Mockito.Times(1)).OnError
(Matchers.Any <Exception>());
// sleep to wait for a few heartbeat calls that can trigger callbacks
Sharpen.Thread.Sleep(50);
// verify no more invocations after the first one.
// ie. callback thread has stopped
Org.Mockito.Mockito.Verify(callbackHandler, Org.Mockito.Mockito.Times(0)).GetProgress
();
Org.Mockito.Mockito.Verify(callbackHandler, Org.Mockito.Mockito.Times(1)).OnError
(Matchers.Any <Exception>());
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestCallAMRMClientAsyncStopFromCallbackHandler()
{
Configuration conf = new Configuration();
TestAMRMClientAsync.TestCallbackHandler2 callbackHandler = new TestAMRMClientAsync.TestCallbackHandler2
(this);
AMRMClient <AMRMClient.ContainerRequest> client = Org.Mockito.Mockito.Mock <AMRMClientImpl
>();
IList <ContainerStatus> completed = Arrays.AsList(ContainerStatus.NewInstance(NewContainerId
(0, 0, 0, 0), ContainerState.Complete, string.Empty, 0));
AllocateResponse response = CreateAllocateResponse(completed, new AList <Container
>(), null);
Org.Mockito.Mockito.When(client.Allocate(Matchers.AnyFloat())).ThenReturn(response
);
AMRMClientAsync <AMRMClient.ContainerRequest> asyncClient = AMRMClientAsync.CreateAMRMClientAsync
(client, 20, callbackHandler);
callbackHandler.asynClient = asyncClient;
asyncClient.Init(conf);
asyncClient.Start();
lock (callbackHandler.notifier)
{
asyncClient.RegisterApplicationMaster("localhost", 1234, null);
while (callbackHandler.notify == false)
{
try
{
Sharpen.Runtime.Wait(callbackHandler.notifier);
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestAMRMClientAsyncShutDown()
{
Configuration conf = new Configuration();
TestAMRMClientAsync.TestCallbackHandler callbackHandler = new TestAMRMClientAsync.TestCallbackHandler
(this);
AMRMClient <AMRMClient.ContainerRequest> client = Org.Mockito.Mockito.Mock <AMRMClientImpl
>();
CreateAllocateResponse(new AList <ContainerStatus>(), new AList <Container>(), null
);
Org.Mockito.Mockito.When(client.Allocate(Matchers.AnyFloat())).ThenThrow(new ApplicationAttemptNotFoundException
("app not found, shut down"));
AMRMClientAsync <AMRMClient.ContainerRequest> asyncClient = AMRMClientAsync.CreateAMRMClientAsync
(client, 10, callbackHandler);
asyncClient.Init(conf);
asyncClient.Start();
asyncClient.RegisterApplicationMaster("localhost", 1234, null);
Sharpen.Thread.Sleep(50);
Org.Mockito.Mockito.Verify(client, Org.Mockito.Mockito.Times(1)).Allocate(Matchers.AnyFloat
());
asyncClient.Stop();
}
/// <exception cref="System.Exception"/>
private void RunHeartBeatThrowOutException(Exception ex)
{
Configuration conf = new Configuration();
TestAMRMClientAsync.TestCallbackHandler callbackHandler = new TestAMRMClientAsync.TestCallbackHandler
(this);
AMRMClient <AMRMClient.ContainerRequest> client = Org.Mockito.Mockito.Mock <AMRMClientImpl
>();
Org.Mockito.Mockito.When(client.Allocate(Matchers.AnyFloat())).ThenThrow(ex);
AMRMClientAsync <AMRMClient.ContainerRequest> asyncClient = AMRMClientAsync.CreateAMRMClientAsync
(client, 20, callbackHandler);
asyncClient.Init(conf);
asyncClient.Start();
lock (callbackHandler.notifier)
{
asyncClient.RegisterApplicationMaster("localhost", 1234, null);
while (callbackHandler.savedException == null)
{
try
{
Sharpen.Runtime.Wait(callbackHandler.notifier);
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
}
}
NUnit.Framework.Assert.IsTrue(callbackHandler.savedException.Message.Contains(ex.
Message));
asyncClient.Stop();
// stopping should have joined all threads and completed all callbacks
NUnit.Framework.Assert.IsTrue(callbackHandler.callbackCount == 0);
}
public virtual void Setup()
{
// start minicluster
conf = new YarnConfiguration();
yarnCluster = new MiniYARNCluster(typeof(TestAMRMClient).FullName, nodeCount, 1,
1);
yarnCluster.Init(conf);
yarnCluster.Start();
NUnit.Framework.Assert.IsNotNull(yarnCluster);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnCluster.GetServiceState
());
// start rm client
yarnClient = (YarnClientImpl)YarnClient.CreateYarnClient();
yarnClient.Init(conf);
yarnClient.Start();
NUnit.Framework.Assert.IsNotNull(yarnClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnClient.GetServiceState
());
// get node info
nodeReports = yarnClient.GetNodeReports(NodeState.Running);
// submit new app
ApplicationSubmissionContext appContext = yarnClient.CreateApplication().GetApplicationSubmissionContext
();
ApplicationId appId = appContext.GetApplicationId();
// set the application name
appContext.SetApplicationName("Test");
// Set the priority for the application master
Priority pri = Priority.NewInstance(0);
appContext.SetPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.SetQueue("default");
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ContainerLaunchContext>();
appContext.SetAMContainerSpec(amContainer);
// unmanaged AM
appContext.SetUnmanagedAM(true);
// Create the request to send to the applications manager
SubmitApplicationRequest appRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<SubmitApplicationRequest>();
appRequest.SetApplicationSubmissionContext(appContext);
// Submit the application to the applications manager
yarnClient.SubmitApplication(appContext);
// wait for app to start
int iterationsLeft = 30;
RMAppAttempt appAttempt = null;
while (iterationsLeft > 0)
{
ApplicationReport appReport = yarnClient.GetApplicationReport(appId);
if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted)
{
attemptId = appReport.GetCurrentApplicationAttemptId();
appAttempt = yarnCluster.GetResourceManager().GetRMContext().GetRMApps()[attemptId
.GetApplicationId()].GetCurrentAppAttempt();
while (true)
{
if (appAttempt.GetAppAttemptState() == RMAppAttemptState.Launched)
{
break;
}
}
break;
}
Sleep(1000);
--iterationsLeft;
}
if (iterationsLeft == 0)
{
NUnit.Framework.Assert.Fail("Application hasn't bee started");
}
// Just dig into the ResourceManager and get the AMRMToken just for the sake
// of testing.
UserGroupInformation.SetLoginUser(UserGroupInformation.CreateRemoteUser(UserGroupInformation
.GetCurrentUser().GetUserName()));
UserGroupInformation.GetCurrentUser().AddToken(appAttempt.GetAMRMToken());
//creating an instance NMTokenCase
nmTokenCache = new NMTokenCache();
// start am rm client
rmClient = (AMRMClientImpl <AMRMClient.ContainerRequest>)AMRMClient.CreateAMRMClient
<AMRMClient.ContainerRequest>();
//setting an instance NMTokenCase
rmClient.SetNMTokenCache(nmTokenCache);
rmClient.Init(conf);
rmClient.Start();
NUnit.Framework.Assert.IsNotNull(rmClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, rmClient.GetServiceState()
);
// start am nm client
nmClient = (NMClientImpl)NMClient.CreateNMClient();
//propagating the AMRMClient NMTokenCache instance
nmClient.SetNMTokenCache(rmClient.GetNMTokenCache());
nmClient.Init(conf);
nmClient.Start();
NUnit.Framework.Assert.IsNotNull(nmClient);
NUnit.Framework.Assert.AreEqual(Service.STATE.Started, nmClient.GetServiceState()
);
}
/// <exception cref="System.Exception"/>
public virtual void TestAMRMClientAsync()
{
Configuration conf = new Configuration();
AtomicBoolean heartbeatBlock = new AtomicBoolean(true);
IList <ContainerStatus> completed1 = Arrays.AsList(ContainerStatus.NewInstance(NewContainerId
(0, 0, 0, 0), ContainerState.Complete, string.Empty, 0));
IList <Container> allocated1 = Arrays.AsList(Container.NewInstance(null, null, null
, null, null, null));
AllocateResponse response1 = CreateAllocateResponse(new AList <ContainerStatus>(),
allocated1, null);
AllocateResponse response2 = CreateAllocateResponse(completed1, new AList <Container
>(), null);
AllocateResponse emptyResponse = CreateAllocateResponse(new AList <ContainerStatus
>(), new AList <Container>(), null);
TestAMRMClientAsync.TestCallbackHandler callbackHandler = new TestAMRMClientAsync.TestCallbackHandler
(this);
AMRMClient <AMRMClient.ContainerRequest> client = Org.Mockito.Mockito.Mock <AMRMClientImpl
>();
AtomicInteger secondHeartbeatSync = new AtomicInteger(0);
Org.Mockito.Mockito.When(client.Allocate(Matchers.AnyFloat())).ThenReturn(response1
).ThenAnswer(new _Answer_89(secondHeartbeatSync, heartbeatBlock, response2)).ThenReturn
(emptyResponse);
Org.Mockito.Mockito.When(client.RegisterApplicationMaster(Matchers.AnyString(), Matchers.AnyInt
(), Matchers.AnyString())).ThenReturn(null);
Org.Mockito.Mockito.When(client.GetAvailableResources()).ThenAnswer(new _Answer_105
(client));
// take client lock to simulate behavior of real impl
AMRMClientAsync <AMRMClient.ContainerRequest> asyncClient = AMRMClientAsync.CreateAMRMClientAsync
(client, 20, callbackHandler);
asyncClient.Init(conf);
asyncClient.Start();
asyncClient.RegisterApplicationMaster("localhost", 1234, null);
// while the CallbackHandler will still only be processing the first response,
// heartbeater thread should still be sending heartbeats.
// To test this, wait for the second heartbeat to be received.
while (secondHeartbeatSync.Get() < 1)
{
Sharpen.Thread.Sleep(10);
}
// heartbeat will be blocked. make sure we can call client methods at this
// time. Checks that heartbeat is not holding onto client lock
System.Diagnostics.Debug.Assert((secondHeartbeatSync.Get() < 2));
asyncClient.GetAvailableResources();
// method returned. now unblock heartbeat
System.Diagnostics.Debug.Assert((secondHeartbeatSync.Get() < 2));
lock (heartbeatBlock)
{
heartbeatBlock.Set(false);
Sharpen.Runtime.NotifyAll(heartbeatBlock);
}
// allocated containers should come before completed containers
NUnit.Framework.Assert.AreEqual(null, callbackHandler.TakeCompletedContainers());
// wait for the allocated containers from the first heartbeat's response
while (callbackHandler.TakeAllocatedContainers() == null)
{
NUnit.Framework.Assert.AreEqual(null, callbackHandler.TakeCompletedContainers());
Sharpen.Thread.Sleep(10);
}
// wait for the completed containers from the second heartbeat's response
while (callbackHandler.TakeCompletedContainers() == null)
{
Sharpen.Thread.Sleep(10);
}
asyncClient.Stop();
NUnit.Framework.Assert.AreEqual(null, callbackHandler.TakeAllocatedContainers());
NUnit.Framework.Assert.AreEqual(null, callbackHandler.TakeCompletedContainers());
}
public _Answer_105(AMRMClient <AMRMClient.ContainerRequest> client)
{
this.client = client;
}