Example #1
0
        /// <exception cref="System.Exception"/>
        public virtual void TestAMRMClientAsyncShutDownWithWaitFor()
        {
            Configuration conf = new Configuration();

            TestAMRMClientAsync.TestCallbackHandler callbackHandler = new TestAMRMClientAsync.TestCallbackHandler
                                                                          (this);
            AMRMClient <AMRMClient.ContainerRequest> client = Org.Mockito.Mockito.Mock <AMRMClientImpl
                                                                                        >();

            Org.Mockito.Mockito.When(client.Allocate(Matchers.AnyFloat())).ThenThrow(new ApplicationAttemptNotFoundException
                                                                                         ("app not found, shut down"));
            AMRMClientAsync <AMRMClient.ContainerRequest> asyncClient = AMRMClientAsync.CreateAMRMClientAsync
                                                                            (client, 10, callbackHandler);

            asyncClient.Init(conf);
            asyncClient.Start();
            Supplier <bool> checker = new _Supplier_246(callbackHandler);

            asyncClient.RegisterApplicationMaster("localhost", 1234, null);
            asyncClient.WaitFor(checker);
            asyncClient.Stop();
            // stopping should have joined all threads and completed all callbacks
            NUnit.Framework.Assert.IsTrue(callbackHandler.callbackCount == 0);
            Org.Mockito.Mockito.Verify(client, Org.Mockito.Mockito.Times(1)).Allocate(Matchers.AnyFloat
                                                                                          ());
            asyncClient.Stop();
        }
Example #2
0
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        public virtual void TestCallAMRMClientAsyncStopFromCallbackHandlerWithWaitFor()
        {
            Configuration conf = new Configuration();

            TestAMRMClientAsync.TestCallbackHandler2 callbackHandler = new TestAMRMClientAsync.TestCallbackHandler2
                                                                           (this);
            AMRMClient <AMRMClient.ContainerRequest> client = Org.Mockito.Mockito.Mock <AMRMClientImpl
                                                                                        >();
            IList <ContainerStatus> completed = Arrays.AsList(ContainerStatus.NewInstance(NewContainerId
                                                                                              (0, 0, 0, 0), ContainerState.Complete, string.Empty, 0));
            AllocateResponse response = CreateAllocateResponse(completed, new AList <Container
                                                                                     >(), null);

            Org.Mockito.Mockito.When(client.Allocate(Matchers.AnyFloat())).ThenReturn(response
                                                                                      );
            AMRMClientAsync <AMRMClient.ContainerRequest> asyncClient = AMRMClientAsync.CreateAMRMClientAsync
                                                                            (client, 20, callbackHandler);

            callbackHandler.asynClient = asyncClient;
            asyncClient.Init(conf);
            asyncClient.Start();
            Supplier <bool> checker = new _Supplier_320(callbackHandler);

            asyncClient.RegisterApplicationMaster("localhost", 1234, null);
            asyncClient.WaitFor(checker);
            NUnit.Framework.Assert.IsTrue(checker.Get());
        }
Example #3
0
 public static Org.Apache.Hadoop.Yarn.Client.Api.Async.AMRMClientAsync <T> CreateAMRMClientAsync
 <T>(AMRMClient <T> client, int intervalMs, AMRMClientAsync.CallbackHandler callbackHandler
     )
     where T : AMRMClient.ContainerRequest
 {
     return(new AMRMClientAsyncImpl <T>(client, intervalMs, callbackHandler));
 }
Example #4
0
 protected internal AMRMClientAsync(AMRMClient <T> client, int intervalMs, AMRMClientAsync.CallbackHandler
                                    callbackHandler)
     : base(typeof(Org.Apache.Hadoop.Yarn.Client.Api.Async.AMRMClientAsync).FullName)
 {
     this.client = client;
     this.heartbeatIntervalMs.Set(intervalMs);
     this.handler = callbackHandler;
 }
Example #5
0
 public AMRMClientAsyncImpl(AMRMClient <T> client, int intervalMs, AMRMClientAsync.CallbackHandler
                            callbackHandler)
     : base(client, intervalMs, callbackHandler)
 {
     heartbeatThread = new AMRMClientAsyncImpl.HeartbeatThread(this);
     handlerThread   = new AMRMClientAsyncImpl.CallbackHandlerThread(this);
     responseQueue   = new LinkedBlockingQueue <AllocateResponse>();
     keepRunning     = true;
     savedException  = null;
 }
Example #6
0
        /// <exception cref="System.Exception"/>
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        /// <exception cref="System.IO.IOException"/>
        internal virtual void RunCallBackThrowOutException(TestAMRMClientAsync.TestCallbackHandler2
                                                           callbackHandler)
        {
            Configuration conf = new Configuration();
            AMRMClient <AMRMClient.ContainerRequest> client = Org.Mockito.Mockito.Mock <AMRMClientImpl
                                                                                        >();
            IList <ContainerStatus> completed = Arrays.AsList(ContainerStatus.NewInstance(NewContainerId
                                                                                              (0, 0, 0, 0), ContainerState.Complete, string.Empty, 0));
            AllocateResponse response = CreateAllocateResponse(completed, new AList <Container
                                                                                     >(), null);

            Org.Mockito.Mockito.When(client.Allocate(Matchers.AnyFloat())).ThenReturn(response
                                                                                      );
            AMRMClientAsync <AMRMClient.ContainerRequest> asyncClient = AMRMClientAsync.CreateAMRMClientAsync
                                                                            (client, 20, callbackHandler);

            callbackHandler.asynClient        = asyncClient;
            callbackHandler.throwOutException = true;
            asyncClient.Init(conf);
            asyncClient.Start();
            // call register and wait for error callback and stop
            lock (callbackHandler.notifier)
            {
                asyncClient.RegisterApplicationMaster("localhost", 1234, null);
                while (callbackHandler.notify == false)
                {
                    try
                    {
                        Sharpen.Runtime.Wait(callbackHandler.notifier);
                    }
                    catch (Exception e)
                    {
                        Sharpen.Runtime.PrintStackTrace(e);
                    }
                }
            }
            // verify error invoked
            Org.Mockito.Mockito.Verify(callbackHandler, Org.Mockito.Mockito.Times(0)).GetProgress
                ();
            Org.Mockito.Mockito.Verify(callbackHandler, Org.Mockito.Mockito.Times(1)).OnError
                (Matchers.Any <Exception>());
            // sleep to wait for a few heartbeat calls that can trigger callbacks
            Sharpen.Thread.Sleep(50);
            // verify no more invocations after the first one.
            // ie. callback thread has stopped
            Org.Mockito.Mockito.Verify(callbackHandler, Org.Mockito.Mockito.Times(0)).GetProgress
                ();
            Org.Mockito.Mockito.Verify(callbackHandler, Org.Mockito.Mockito.Times(1)).OnError
                (Matchers.Any <Exception>());
        }
Example #7
0
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        public virtual void TestCallAMRMClientAsyncStopFromCallbackHandler()
        {
            Configuration conf = new Configuration();

            TestAMRMClientAsync.TestCallbackHandler2 callbackHandler = new TestAMRMClientAsync.TestCallbackHandler2
                                                                           (this);
            AMRMClient <AMRMClient.ContainerRequest> client = Org.Mockito.Mockito.Mock <AMRMClientImpl
                                                                                        >();
            IList <ContainerStatus> completed = Arrays.AsList(ContainerStatus.NewInstance(NewContainerId
                                                                                              (0, 0, 0, 0), ContainerState.Complete, string.Empty, 0));
            AllocateResponse response = CreateAllocateResponse(completed, new AList <Container
                                                                                     >(), null);

            Org.Mockito.Mockito.When(client.Allocate(Matchers.AnyFloat())).ThenReturn(response
                                                                                      );
            AMRMClientAsync <AMRMClient.ContainerRequest> asyncClient = AMRMClientAsync.CreateAMRMClientAsync
                                                                            (client, 20, callbackHandler);

            callbackHandler.asynClient = asyncClient;
            asyncClient.Init(conf);
            asyncClient.Start();
            lock (callbackHandler.notifier)
            {
                asyncClient.RegisterApplicationMaster("localhost", 1234, null);
                while (callbackHandler.notify == false)
                {
                    try
                    {
                        Sharpen.Runtime.Wait(callbackHandler.notifier);
                    }
                    catch (Exception e)
                    {
                        Sharpen.Runtime.PrintStackTrace(e);
                    }
                }
            }
        }
Example #8
0
        /// <exception cref="System.Exception"/>
        public virtual void TestAMRMClientAsyncShutDown()
        {
            Configuration conf = new Configuration();

            TestAMRMClientAsync.TestCallbackHandler callbackHandler = new TestAMRMClientAsync.TestCallbackHandler
                                                                          (this);
            AMRMClient <AMRMClient.ContainerRequest> client = Org.Mockito.Mockito.Mock <AMRMClientImpl
                                                                                        >();

            CreateAllocateResponse(new AList <ContainerStatus>(), new AList <Container>(), null
                                   );
            Org.Mockito.Mockito.When(client.Allocate(Matchers.AnyFloat())).ThenThrow(new ApplicationAttemptNotFoundException
                                                                                         ("app not found, shut down"));
            AMRMClientAsync <AMRMClient.ContainerRequest> asyncClient = AMRMClientAsync.CreateAMRMClientAsync
                                                                            (client, 10, callbackHandler);

            asyncClient.Init(conf);
            asyncClient.Start();
            asyncClient.RegisterApplicationMaster("localhost", 1234, null);
            Sharpen.Thread.Sleep(50);
            Org.Mockito.Mockito.Verify(client, Org.Mockito.Mockito.Times(1)).Allocate(Matchers.AnyFloat
                                                                                          ());
            asyncClient.Stop();
        }
Example #9
0
        /// <exception cref="System.Exception"/>
        private void RunHeartBeatThrowOutException(Exception ex)
        {
            Configuration conf = new Configuration();

            TestAMRMClientAsync.TestCallbackHandler callbackHandler = new TestAMRMClientAsync.TestCallbackHandler
                                                                          (this);
            AMRMClient <AMRMClient.ContainerRequest> client = Org.Mockito.Mockito.Mock <AMRMClientImpl
                                                                                        >();

            Org.Mockito.Mockito.When(client.Allocate(Matchers.AnyFloat())).ThenThrow(ex);
            AMRMClientAsync <AMRMClient.ContainerRequest> asyncClient = AMRMClientAsync.CreateAMRMClientAsync
                                                                            (client, 20, callbackHandler);

            asyncClient.Init(conf);
            asyncClient.Start();
            lock (callbackHandler.notifier)
            {
                asyncClient.RegisterApplicationMaster("localhost", 1234, null);
                while (callbackHandler.savedException == null)
                {
                    try
                    {
                        Sharpen.Runtime.Wait(callbackHandler.notifier);
                    }
                    catch (Exception e)
                    {
                        Sharpen.Runtime.PrintStackTrace(e);
                    }
                }
            }
            NUnit.Framework.Assert.IsTrue(callbackHandler.savedException.Message.Contains(ex.
                                                                                          Message));
            asyncClient.Stop();
            // stopping should have joined all threads and completed all callbacks
            NUnit.Framework.Assert.IsTrue(callbackHandler.callbackCount == 0);
        }
Example #10
0
        public virtual void Setup()
        {
            // start minicluster
            conf        = new YarnConfiguration();
            yarnCluster = new MiniYARNCluster(typeof(TestAMRMClient).FullName, nodeCount, 1,
                                              1);
            yarnCluster.Init(conf);
            yarnCluster.Start();
            NUnit.Framework.Assert.IsNotNull(yarnCluster);
            NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnCluster.GetServiceState
                                                ());
            // start rm client
            yarnClient = (YarnClientImpl)YarnClient.CreateYarnClient();
            yarnClient.Init(conf);
            yarnClient.Start();
            NUnit.Framework.Assert.IsNotNull(yarnClient);
            NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnClient.GetServiceState
                                                ());
            // get node info
            nodeReports = yarnClient.GetNodeReports(NodeState.Running);
            // submit new app
            ApplicationSubmissionContext appContext = yarnClient.CreateApplication().GetApplicationSubmissionContext
                                                          ();
            ApplicationId appId = appContext.GetApplicationId();

            // set the application name
            appContext.SetApplicationName("Test");
            // Set the priority for the application master
            Priority pri = Priority.NewInstance(0);

            appContext.SetPriority(pri);
            // Set the queue to which this application is to be submitted in the RM
            appContext.SetQueue("default");
            // Set up the container launch context for the application master
            ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                                 <ContainerLaunchContext>();

            appContext.SetAMContainerSpec(amContainer);
            // unmanaged AM
            appContext.SetUnmanagedAM(true);
            // Create the request to send to the applications manager
            SubmitApplicationRequest appRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                                  <SubmitApplicationRequest>();

            appRequest.SetApplicationSubmissionContext(appContext);
            // Submit the application to the applications manager
            yarnClient.SubmitApplication(appContext);
            // wait for app to start
            int          iterationsLeft = 30;
            RMAppAttempt appAttempt     = null;

            while (iterationsLeft > 0)
            {
                ApplicationReport appReport = yarnClient.GetApplicationReport(appId);
                if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted)
                {
                    attemptId  = appReport.GetCurrentApplicationAttemptId();
                    appAttempt = yarnCluster.GetResourceManager().GetRMContext().GetRMApps()[attemptId
                                                                                             .GetApplicationId()].GetCurrentAppAttempt();
                    while (true)
                    {
                        if (appAttempt.GetAppAttemptState() == RMAppAttemptState.Launched)
                        {
                            break;
                        }
                    }
                    break;
                }
                Sleep(1000);
                --iterationsLeft;
            }
            if (iterationsLeft == 0)
            {
                NUnit.Framework.Assert.Fail("Application hasn't bee started");
            }
            // Just dig into the ResourceManager and get the AMRMToken just for the sake
            // of testing.
            UserGroupInformation.SetLoginUser(UserGroupInformation.CreateRemoteUser(UserGroupInformation
                                                                                    .GetCurrentUser().GetUserName()));
            UserGroupInformation.GetCurrentUser().AddToken(appAttempt.GetAMRMToken());
            //creating an instance NMTokenCase
            nmTokenCache = new NMTokenCache();
            // start am rm client
            rmClient = (AMRMClientImpl <AMRMClient.ContainerRequest>)AMRMClient.CreateAMRMClient
                       <AMRMClient.ContainerRequest>();
            //setting an instance NMTokenCase
            rmClient.SetNMTokenCache(nmTokenCache);
            rmClient.Init(conf);
            rmClient.Start();
            NUnit.Framework.Assert.IsNotNull(rmClient);
            NUnit.Framework.Assert.AreEqual(Service.STATE.Started, rmClient.GetServiceState()
                                            );
            // start am nm client
            nmClient = (NMClientImpl)NMClient.CreateNMClient();
            //propagating the AMRMClient NMTokenCache instance
            nmClient.SetNMTokenCache(rmClient.GetNMTokenCache());
            nmClient.Init(conf);
            nmClient.Start();
            NUnit.Framework.Assert.IsNotNull(nmClient);
            NUnit.Framework.Assert.AreEqual(Service.STATE.Started, nmClient.GetServiceState()
                                            );
        }
Example #11
0
        /// <exception cref="System.Exception"/>
        public virtual void TestAMRMClientAsync()
        {
            Configuration           conf           = new Configuration();
            AtomicBoolean           heartbeatBlock = new AtomicBoolean(true);
            IList <ContainerStatus> completed1     = Arrays.AsList(ContainerStatus.NewInstance(NewContainerId
                                                                                                   (0, 0, 0, 0), ContainerState.Complete, string.Empty, 0));
            IList <Container> allocated1 = Arrays.AsList(Container.NewInstance(null, null, null
                                                                               , null, null, null));
            AllocateResponse response1 = CreateAllocateResponse(new AList <ContainerStatus>(),
                                                                allocated1, null);
            AllocateResponse response2 = CreateAllocateResponse(completed1, new AList <Container
                                                                                       >(), null);
            AllocateResponse emptyResponse = CreateAllocateResponse(new AList <ContainerStatus
                                                                               >(), new AList <Container>(), null);

            TestAMRMClientAsync.TestCallbackHandler callbackHandler = new TestAMRMClientAsync.TestCallbackHandler
                                                                          (this);
            AMRMClient <AMRMClient.ContainerRequest> client = Org.Mockito.Mockito.Mock <AMRMClientImpl
                                                                                        >();
            AtomicInteger secondHeartbeatSync = new AtomicInteger(0);

            Org.Mockito.Mockito.When(client.Allocate(Matchers.AnyFloat())).ThenReturn(response1
                                                                                      ).ThenAnswer(new _Answer_89(secondHeartbeatSync, heartbeatBlock, response2)).ThenReturn
                (emptyResponse);
            Org.Mockito.Mockito.When(client.RegisterApplicationMaster(Matchers.AnyString(), Matchers.AnyInt
                                                                          (), Matchers.AnyString())).ThenReturn(null);
            Org.Mockito.Mockito.When(client.GetAvailableResources()).ThenAnswer(new _Answer_105
                                                                                    (client));
            // take client lock to simulate behavior of real impl
            AMRMClientAsync <AMRMClient.ContainerRequest> asyncClient = AMRMClientAsync.CreateAMRMClientAsync
                                                                            (client, 20, callbackHandler);

            asyncClient.Init(conf);
            asyncClient.Start();
            asyncClient.RegisterApplicationMaster("localhost", 1234, null);
            // while the CallbackHandler will still only be processing the first response,
            // heartbeater thread should still be sending heartbeats.
            // To test this, wait for the second heartbeat to be received.
            while (secondHeartbeatSync.Get() < 1)
            {
                Sharpen.Thread.Sleep(10);
            }
            // heartbeat will be blocked. make sure we can call client methods at this
            // time. Checks that heartbeat is not holding onto client lock
            System.Diagnostics.Debug.Assert((secondHeartbeatSync.Get() < 2));
            asyncClient.GetAvailableResources();
            // method returned. now unblock heartbeat
            System.Diagnostics.Debug.Assert((secondHeartbeatSync.Get() < 2));
            lock (heartbeatBlock)
            {
                heartbeatBlock.Set(false);
                Sharpen.Runtime.NotifyAll(heartbeatBlock);
            }
            // allocated containers should come before completed containers
            NUnit.Framework.Assert.AreEqual(null, callbackHandler.TakeCompletedContainers());
            // wait for the allocated containers from the first heartbeat's response
            while (callbackHandler.TakeAllocatedContainers() == null)
            {
                NUnit.Framework.Assert.AreEqual(null, callbackHandler.TakeCompletedContainers());
                Sharpen.Thread.Sleep(10);
            }
            // wait for the completed containers from the second heartbeat's response
            while (callbackHandler.TakeCompletedContainers() == null)
            {
                Sharpen.Thread.Sleep(10);
            }
            asyncClient.Stop();
            NUnit.Framework.Assert.AreEqual(null, callbackHandler.TakeAllocatedContainers());
            NUnit.Framework.Assert.AreEqual(null, callbackHandler.TakeCompletedContainers());
        }
Example #12
0
 public _Answer_105(AMRMClient <AMRMClient.ContainerRequest> client)
 {
     this.client = client;
 }