Ejemplo n.º 1
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        public override JobStatus SubmitJob(JobID jobId, string jobSubmitDir, Credentials
                                            ts)
        {
            AddHistoryToken(ts);
            // Construct necessary information to start the MR AM
            ApplicationSubmissionContext appContext = CreateApplicationSubmissionContext(conf
                                                                                         , jobSubmitDir, ts);

            // Submit to ResourceManager
            try
            {
                ApplicationId     applicationId = resMgrDelegate.SubmitApplication(appContext);
                ApplicationReport appMaster     = resMgrDelegate.GetApplicationReport(applicationId);
                string            diagnostics   = (appMaster == null ? "application report is null" : appMaster
                                                   .GetDiagnostics());
                if (appMaster == null || appMaster.GetYarnApplicationState() == YarnApplicationState
                    .Failed || appMaster.GetYarnApplicationState() == YarnApplicationState.Killed)
                {
                    throw new IOException("Failed to run job : " + diagnostics);
                }
                return(clientCache.GetClient(jobId).GetJobStatus(jobId));
            }
            catch (YarnException e)
            {
                throw new IOException(e);
            }
        }
Ejemplo n.º 2
0
 /// <exception cref="System.Exception"/>
 public virtual void TestJobKill()
 {
     clientDelegate = Org.Mockito.Mockito.Mock <ClientServiceDelegate>();
     Org.Mockito.Mockito.When(clientDelegate.GetJobStatus(Matchers.Any <JobID>())).ThenReturn
         (new JobStatus(jobId, 0f, 0f, 0f, 0f, JobStatus.State.Prep, JobPriority.High, "tmp"
                        , "tmp", "tmp", "tmp"));
     Org.Mockito.Mockito.When(clientDelegate.KillJob(Matchers.Any <JobID>())).ThenReturn
         (true);
     Org.Mockito.Mockito.DoAnswer(new _Answer_177(this)).When(clientCache).GetClient(Matchers.Any
                                                                                     <JobID>());
     yarnRunner.KillJob(jobId);
     Org.Mockito.Mockito.Verify(resourceMgrDelegate).KillApplication(appId);
     Org.Mockito.Mockito.When(clientDelegate.GetJobStatus(Matchers.Any <JobID>())).ThenReturn
         (new JobStatus(jobId, 0f, 0f, 0f, 0f, JobStatus.State.Running, JobPriority.High,
                        "tmp", "tmp", "tmp", "tmp"));
     yarnRunner.KillJob(jobId);
     Org.Mockito.Mockito.Verify(clientDelegate).KillJob(jobId);
     Org.Mockito.Mockito.When(clientDelegate.GetJobStatus(Matchers.Any <JobID>())).ThenReturn
         (null);
     Org.Mockito.Mockito.When(resourceMgrDelegate.GetApplicationReport(Matchers.Any <ApplicationId
                                                                                     >())).ThenReturn(ApplicationReport.NewInstance(appId, null, "tmp", "tmp", "tmp",
                                                                                                                                    "tmp", 0, null, YarnApplicationState.Finished, "tmp", "tmp", 0l, 0l, FinalApplicationStatus
                                                                                                                                    .Succeeded, null, null, 0f, "tmp", null));
     yarnRunner.KillJob(jobId);
     Org.Mockito.Mockito.Verify(clientDelegate).KillJob(jobId);
 }
Ejemplo n.º 3
0
        public virtual void TestRMDownRestoreForJobStatusBeforeGetAMReport()
        {
            Configuration conf = new YarnConfiguration();

            conf.SetInt(MRJobConfig.MrClientMaxRetries, 3);
            conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
            conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient);
            MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
                );

            Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
                                                                                   >())).ThenReturn(GetJobReportResponse());
            ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();

            try
            {
                Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenThrow
                    (new UndeclaredThrowableException(new IOException("Connection refuced1"))).ThenThrow
                    (new UndeclaredThrowableException(new IOException("Connection refuced2"))).ThenReturn
                    (GetFinishedApplicationReport());
                ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rmDelegate
                                                                                        , oldJobId, historyServerProxy);
                JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
                Org.Mockito.Mockito.Verify(rmDelegate, Org.Mockito.Mockito.Times(3)).GetApplicationReport
                    (Matchers.Any <ApplicationId>());
                NUnit.Framework.Assert.IsNotNull(jobStatus);
            }
            catch (YarnException e)
            {
                throw new IOException(e);
            }
        }
Ejemplo n.º 4
0
        /// <exception cref="System.IO.IOException"/>
        private void TestRMDownForJobStatusBeforeGetAMReport(Configuration conf, int noOfRetries
                                                             )
        {
            conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
            conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient);
            MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
                );
            ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();

            try
            {
                Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenThrow
                    (new UndeclaredThrowableException(new IOException("Connection refuced1"))).ThenThrow
                    (new UndeclaredThrowableException(new IOException("Connection refuced2"))).ThenThrow
                    (new UndeclaredThrowableException(new IOException("Connection refuced3")));
                ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rmDelegate
                                                                                        , oldJobId, historyServerProxy);
                try
                {
                    clientServiceDelegate.GetJobStatus(oldJobId);
                    NUnit.Framework.Assert.Fail("It should throw exception after retries");
                }
                catch (IOException e)
                {
                    System.Console.Out.WriteLine("fail to get job status,and e=" + e.ToString());
                }
                Org.Mockito.Mockito.Verify(rmDelegate, Org.Mockito.Mockito.Times(noOfRetries)).GetApplicationReport
                    (Matchers.Any <ApplicationId>());
            }
            catch (YarnException e)
            {
                throw new IOException(e);
            }
        }
Ejemplo n.º 5
0
        public virtual void TestRetriesOnAMConnectionFailures()
        {
            if (!isAMReachableFromClient)
            {
                return;
            }
            ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();

            Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
                                                                 ())).ThenReturn(GetRunningApplicationReport("am1", 78));
            // throw exception in 1st, 2nd, 3rd and 4th call of getJobReport, and
            // succeed in the 5th call.
            MRClientProtocol amProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>();

            Org.Mockito.Mockito.When(amProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
                                                                        >())).ThenThrow(new RuntimeException("11")).ThenThrow(new RuntimeException("22")
                                                                                                                              ).ThenThrow(new RuntimeException("33")).ThenThrow(new RuntimeException("44")).ThenReturn
                (GetJobReportResponse());
            Configuration conf = new YarnConfiguration();

            conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
            conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient);
            ClientServiceDelegate clientServiceDelegate = new _ClientServiceDelegate_167(amProxy
                                                                                         , conf, rm, oldJobId, null);
            JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);

            NUnit.Framework.Assert.IsNotNull(jobStatus);
            // assert maxClientRetry is not decremented.
            NUnit.Framework.Assert.AreEqual(conf.GetInt(MRJobConfig.MrClientMaxRetries, MRJobConfig
                                                        .DefaultMrClientMaxRetries), clientServiceDelegate.GetMaxClientRetry());
            Org.Mockito.Mockito.Verify(amProxy, Org.Mockito.Mockito.Times(5)).GetJobReport(Matchers.Any
                                                                                           <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest>());
        }
Ejemplo n.º 6
0
        public virtual void TestAMAccessDisabled()
        {
            //test only applicable when AM not reachable
            if (isAMReachableFromClient)
            {
                return;
            }
            MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
                );

            Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())).
            ThenReturn(GetJobReportResponseFromHistoryServer());
            ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();

            try
            {
                Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenReturn
                    (GetRunningApplicationReport("am1", 78)).ThenReturn(GetRunningApplicationReport(
                                                                            "am1", 78)).ThenReturn(GetRunningApplicationReport("am1", 78)).ThenReturn(GetFinishedApplicationReport
                                                                                                                                                          ());
            }
            catch (YarnException e)
            {
                throw new IOException(e);
            }
            ClientServiceDelegate clientServiceDelegate = Org.Mockito.Mockito.Spy(GetClientServiceDelegate
                                                                                      (historyServerProxy, rmDelegate));
            JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);

            NUnit.Framework.Assert.IsNotNull(jobStatus);
            NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetJobName());
            Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy
                (Matchers.Any <IPEndPoint>());
            // Should not reach AM even for second and third times too.
            jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
            NUnit.Framework.Assert.IsNotNull(jobStatus);
            NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetJobName());
            Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy
                (Matchers.Any <IPEndPoint>());
            jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
            NUnit.Framework.Assert.IsNotNull(jobStatus);
            NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetJobName());
            Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy
                (Matchers.Any <IPEndPoint>());
            // The third time around, app is completed, so should go to JHS
            JobStatus jobStatus1 = clientServiceDelegate.GetJobStatus(oldJobId);

            NUnit.Framework.Assert.IsNotNull(jobStatus1);
            NUnit.Framework.Assert.AreEqual("TestJobFilePath", jobStatus1.GetJobFile());
            NUnit.Framework.Assert.AreEqual("http://TestTrackingUrl", jobStatus1.GetTrackingUrl
                                                ());
            NUnit.Framework.Assert.AreEqual(1.0f, jobStatus1.GetMapProgress(), 0.0f);
            NUnit.Framework.Assert.AreEqual(1.0f, jobStatus1.GetReduceProgress(), 0.0f);
            Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy
                (Matchers.Any <IPEndPoint>());
        }
Ejemplo n.º 7
0
        /// <exception cref="System.IO.IOException"/>
        private ResourceMgrDelegate GetRMDelegate()
        {
            ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();

            try
            {
                ApplicationId appId = jobId.GetAppId();
                Org.Mockito.Mockito.When(rm.GetApplicationReport(appId)).ThenThrow(new ApplicationNotFoundException
                                                                                       (appId + " not found"));
            }
            catch (YarnException e)
            {
                throw new IOException(e);
            }
            return(rm);
        }
Ejemplo n.º 8
0
        public virtual void TestCountersFromHistoryServer()
        {
            MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
                );

            Org.Mockito.Mockito.When(historyServerProxy.GetCounters(GetCountersRequest())).ThenReturn
                (GetCountersResponseFromHistoryServer());
            ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();

            Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
                                                                 ())).ThenReturn(null);
            ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy
                                                                                   , rm);
            Counters counters = TypeConverter.ToYarn(clientServiceDelegate.GetJobCounters(oldJobId
                                                                                          ));

            NUnit.Framework.Assert.IsNotNull(counters);
            NUnit.Framework.Assert.AreEqual(1001, counters.GetCounterGroup("dummyCounters").GetCounter
                                                ("dummyCounter").GetValue());
        }
Ejemplo n.º 9
0
        public virtual void TestRetriesOnConnectionFailure()
        {
            MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
                );

            Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())).
            ThenThrow(new RuntimeException("1")).ThenThrow(new RuntimeException("2")).ThenReturn
                (GetJobReportResponse());
            ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();

            Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
                                                                 ())).ThenReturn(null);
            ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy
                                                                                   , rm);
            JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);

            NUnit.Framework.Assert.IsNotNull(jobStatus);
            Org.Mockito.Mockito.Verify(historyServerProxy, Org.Mockito.Mockito.Times(3)).GetJobReport
                (Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
                               >());
        }
Ejemplo n.º 10
0
        public virtual void TestHistoryServerNotConfigured()
        {
            //RM doesn't have app report and job History Server is not configured
            ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(null, GetRMDelegate
                                                                                       ());
            JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);

            NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetUsername());
            NUnit.Framework.Assert.AreEqual(JobStatus.State.Prep, jobStatus.GetState());
            //RM has app report and job History Server is not configured
            ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
            ApplicationReport   applicationReport = GetFinishedApplicationReport();

            Org.Mockito.Mockito.When(rm.GetApplicationReport(jobId.GetAppId())).ThenReturn(applicationReport
                                                                                           );
            clientServiceDelegate = GetClientServiceDelegate(null, rm);
            jobStatus             = clientServiceDelegate.GetJobStatus(oldJobId);
            NUnit.Framework.Assert.AreEqual(applicationReport.GetUser(), jobStatus.GetUsername
                                                ());
            NUnit.Framework.Assert.AreEqual(JobStatus.State.Succeeded, jobStatus.GetState());
        }
Ejemplo n.º 11
0
        public virtual void TestJobReportFromHistoryServer()
        {
            MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
                );

            Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())).
            ThenReturn(GetJobReportResponseFromHistoryServer());
            ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();

            Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
                                                                 ())).ThenReturn(null);
            ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy
                                                                                   , rm);
            JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);

            NUnit.Framework.Assert.IsNotNull(jobStatus);
            NUnit.Framework.Assert.AreEqual("TestJobFilePath", jobStatus.GetJobFile());
            NUnit.Framework.Assert.AreEqual("http://TestTrackingUrl", jobStatus.GetTrackingUrl
                                                ());
            NUnit.Framework.Assert.AreEqual(1.0f, jobStatus.GetMapProgress(), 0.0f);
            NUnit.Framework.Assert.AreEqual(1.0f, jobStatus.GetReduceProgress(), 0.0f);
        }
Ejemplo n.º 12
0
        public virtual void TestNoRetryOnAMAuthorizationException()
        {
            if (!isAMReachableFromClient)
            {
                return;
            }
            ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();

            Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
                                                                 ())).ThenReturn(GetRunningApplicationReport("am1", 78));
            // throw authorization exception on first invocation
            MRClientProtocol amProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>();

            Org.Mockito.Mockito.When(amProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
                                                                        >())).ThenThrow(new AuthorizationException("Denied"));
            Configuration conf = new YarnConfiguration();

            conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
            conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient);
            ClientServiceDelegate clientServiceDelegate = new _ClientServiceDelegate_205(amProxy
                                                                                         , conf, rm, oldJobId, null);

            try
            {
                clientServiceDelegate.GetJobStatus(oldJobId);
                NUnit.Framework.Assert.Fail("Exception should be thrown upon AuthorizationException"
                                            );
            }
            catch (IOException e)
            {
                NUnit.Framework.Assert.AreEqual(typeof(AuthorizationException).FullName + ": Denied"
                                                , e.Message);
            }
            // assert maxClientRetry is not decremented.
            NUnit.Framework.Assert.AreEqual(conf.GetInt(MRJobConfig.MrClientMaxRetries, MRJobConfig
                                                        .DefaultMrClientMaxRetries), clientServiceDelegate.GetMaxClientRetry());
            Org.Mockito.Mockito.Verify(amProxy, Org.Mockito.Mockito.Times(1)).GetJobReport(Matchers.Any
                                                                                           <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest>());
        }
Ejemplo n.º 13
0
        public virtual void TestRemoteExceptionFromHistoryServer()
        {
            MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
                );

            Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())).
            ThenThrow(new IOException("Job ID doesnot Exist"));
            ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();

            Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
                                                                 ())).ThenReturn(null);
            ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy
                                                                                   , rm);

            try
            {
                clientServiceDelegate.GetJobStatus(oldJobId);
                NUnit.Framework.Assert.Fail("Invoke should throw exception after retries.");
            }
            catch (IOException e)
            {
                NUnit.Framework.Assert.IsTrue(e.Message.Contains("Job ID doesnot Exist"));
            }
        }
Ejemplo n.º 14
0
        /// <exception cref="System.IO.IOException"/>
        private MRClientProtocol GetProxy()
        {
            if (realProxy != null)
            {
                return(realProxy);
            }
            // Possibly allow nulls through the PB tunnel, otherwise deal with an exception
            // and redirect to the history server.
            ApplicationReport application = null;

            try
            {
                application = rm.GetApplicationReport(appId);
            }
            catch (ApplicationNotFoundException)
            {
                application = null;
            }
            catch (YarnException e2)
            {
                throw new IOException(e2);
            }
            if (application != null)
            {
                trackingUrl = application.GetTrackingUrl();
            }
            IPEndPoint serviceAddr = null;

            while (application == null || YarnApplicationState.Running == application.GetYarnApplicationState
                       ())
            {
                if (application == null)
                {
                    Log.Info("Could not get Job info from RM for job " + jobId + ". Redirecting to job history server."
                             );
                    return(CheckAndGetHSProxy(null, JobState.New));
                }
                try
                {
                    if (application.GetHost() == null || string.Empty.Equals(application.GetHost()))
                    {
                        Log.Debug("AM not assigned to Job. Waiting to get the AM ...");
                        Sharpen.Thread.Sleep(2000);
                        Log.Debug("Application state is " + application.GetYarnApplicationState());
                        application = rm.GetApplicationReport(appId);
                        continue;
                    }
                    else
                    {
                        if (Unavailable.Equals(application.GetHost()))
                        {
                            if (!amAclDisabledStatusLogged)
                            {
                                Log.Info("Job " + jobId + " is running, but the host is unknown." + " Verify user has VIEW_JOB access."
                                         );
                                amAclDisabledStatusLogged = true;
                            }
                            return(GetNotRunningJob(application, JobState.Running));
                        }
                    }
                    if (!conf.GetBoolean(MRJobConfig.JobAmAccessDisabled, false))
                    {
                        UserGroupInformation newUgi = UserGroupInformation.CreateRemoteUser(UserGroupInformation
                                                                                            .GetCurrentUser().GetUserName());
                        serviceAddr = NetUtils.CreateSocketAddrForHost(application.GetHost(), application
                                                                       .GetRpcPort());
                        if (UserGroupInformation.IsSecurityEnabled())
                        {
                            Token clientToAMToken = application.GetClientToAMToken();
                            Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> token = ConverterUtils
                                                                                                       .ConvertFromYarn(clientToAMToken, serviceAddr);
                            newUgi.AddToken(token);
                        }
                        Log.Debug("Connecting to " + serviceAddr);
                        IPEndPoint finalServiceAddr = serviceAddr;
                        realProxy = newUgi.DoAs(new _PrivilegedExceptionAction_202(this, finalServiceAddr
                                                                                   ));
                    }
                    else
                    {
                        if (!amAclDisabledStatusLogged)
                        {
                            Log.Info("Network ACL closed to AM for job " + jobId + ". Not going to try to reach the AM."
                                     );
                            amAclDisabledStatusLogged = true;
                        }
                        return(GetNotRunningJob(null, JobState.Running));
                    }
                    return(realProxy);
                }
                catch (IOException)
                {
                    //possibly the AM has crashed
                    //there may be some time before AM is restarted
                    //keep retrying by getting the address from RM
                    Log.Info("Could not connect to " + serviceAddr + ". Waiting for getting the latest AM address..."
                             );
                    try
                    {
                        Sharpen.Thread.Sleep(2000);
                    }
                    catch (Exception e1)
                    {
                        Log.Warn("getProxy() call interruped", e1);
                        throw new YarnRuntimeException(e1);
                    }
                    try
                    {
                        application = rm.GetApplicationReport(appId);
                    }
                    catch (YarnException e1)
                    {
                        throw new IOException(e1);
                    }
                    if (application == null)
                    {
                        Log.Info("Could not get Job info from RM for job " + jobId + ". Redirecting to job history server."
                                 );
                        return(CheckAndGetHSProxy(null, JobState.Running));
                    }
                }
                catch (Exception e)
                {
                    Log.Warn("getProxy() call interruped", e);
                    throw new YarnRuntimeException(e);
                }
                catch (YarnException e)
                {
                    throw new IOException(e);
                }
            }
            string user = application.GetUser();

            if (user == null)
            {
                throw new IOException("User is not set in the application report");
            }
            if (application.GetYarnApplicationState() == YarnApplicationState.New || application
                .GetYarnApplicationState() == YarnApplicationState.NewSaving || application.GetYarnApplicationState
                    () == YarnApplicationState.Submitted || application.GetYarnApplicationState() ==
                YarnApplicationState.Accepted)
            {
                realProxy = null;
                return(GetNotRunningJob(application, JobState.New));
            }
            if (application.GetYarnApplicationState() == YarnApplicationState.Failed)
            {
                realProxy = null;
                return(GetNotRunningJob(application, JobState.Failed));
            }
            if (application.GetYarnApplicationState() == YarnApplicationState.Killed)
            {
                realProxy = null;
                return(GetNotRunningJob(application, JobState.Killed));
            }
            //History server can serve a job only if application
            //succeeded.
            if (application.GetYarnApplicationState() == YarnApplicationState.Finished)
            {
                Log.Info("Application state is completed. FinalApplicationStatus=" + application.
                         GetFinalApplicationStatus().ToString() + ". Redirecting to job history server");
                realProxy = CheckAndGetHSProxy(application, JobState.Succeeded);
            }
            return(realProxy);
        }
Ejemplo n.º 15
0
        public virtual void TestReconnectOnAMRestart()
        {
            //test not applicable when AM not reachable
            //as instantiateAMProxy is not called at all
            if (!isAMReachableFromClient)
            {
                return;
            }
            MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
                );
            // RM returns AM1 url, null, null and AM2 url on invocations.
            // Nulls simulate the time when AM2 is in the process of restarting.
            ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();

            try
            {
                Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenReturn
                    (GetRunningApplicationReport("am1", 78)).ThenReturn(GetRunningApplicationReport(
                                                                            null, 0)).ThenReturn(GetRunningApplicationReport(null, 0)).ThenReturn(GetRunningApplicationReport
                                                                                                                                                      ("am2", 90));
            }
            catch (YarnException e)
            {
                throw new IOException(e);
            }
            Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse jobReportResponse1
                = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse
                                            >();
            Org.Mockito.Mockito.When(jobReportResponse1.GetJobReport()).ThenReturn(MRBuilderUtils
                                                                                   .NewJobReport(jobId, "jobName-firstGen", "user", JobState.Running, 0, 0, 0, 0, 0
                                                                                                 , 0, 0, "anything", null, false, string.Empty));
            // First AM returns a report with jobName firstGen and simulates AM shutdown
            // on second invocation.
            MRClientProtocol firstGenAMProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>();

            Org.Mockito.Mockito.When(firstGenAMProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
                                                                                >())).ThenReturn(jobReportResponse1).ThenThrow(new RuntimeException("AM is down!"
                                                                                                                                                    ));
            Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse jobReportResponse2
                = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse
                                            >();
            Org.Mockito.Mockito.When(jobReportResponse2.GetJobReport()).ThenReturn(MRBuilderUtils
                                                                                   .NewJobReport(jobId, "jobName-secondGen", "user", JobState.Running, 0, 0, 0, 0,
                                                                                                 0, 0, 0, "anything", null, false, string.Empty));
            // Second AM generation returns a report with jobName secondGen
            MRClientProtocol secondGenAMProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>();

            Org.Mockito.Mockito.When(secondGenAMProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
                                                                                 >())).ThenReturn(jobReportResponse2);
            ClientServiceDelegate clientServiceDelegate = Org.Mockito.Mockito.Spy(GetClientServiceDelegate
                                                                                      (historyServerProxy, rmDelegate));

            // First time, connection should be to AM1, then to AM2. Further requests
            // should use the same proxy to AM2 and so instantiateProxy shouldn't be
            // called.
            Org.Mockito.Mockito.DoReturn(firstGenAMProxy).DoReturn(secondGenAMProxy).When(clientServiceDelegate
                                                                                          ).InstantiateAMProxy(Matchers.Any <IPEndPoint>());
            JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);

            NUnit.Framework.Assert.IsNotNull(jobStatus);
            NUnit.Framework.Assert.AreEqual("jobName-firstGen", jobStatus.GetJobName());
            jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
            NUnit.Framework.Assert.IsNotNull(jobStatus);
            NUnit.Framework.Assert.AreEqual("jobName-secondGen", jobStatus.GetJobName());
            jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
            NUnit.Framework.Assert.IsNotNull(jobStatus);
            NUnit.Framework.Assert.AreEqual("jobName-secondGen", jobStatus.GetJobName());
            Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(2)).InstantiateAMProxy
                (Matchers.Any <IPEndPoint>());
        }